diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index 145d742d5b12..cecbd4515c8c 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -15,6 +15,9 @@ outputs: python: description: Run Python tests / ruff / ty / windows-footguns. value: ${{ steps.classify.outputs.python }} + python_prod: + description: Python changes outside tests/ — gates product jobs (Desktop E2E, Docker). + value: ${{ steps.classify.outputs.python_prod }} frontend: description: Run the TypeScript testing matrix + desktop build. value: ${{ steps.classify.outputs.frontend }} diff --git a/.github/actions/get-app-token/action.yml b/.github/actions/get-app-token/action.yml index 0b01cf900146..2aaf303ab2de 100644 --- a/.github/actions/get-app-token/action.yml +++ b/.github/actions/get-app-token/action.yml @@ -45,9 +45,8 @@ runs: shell: bash env: CLIENT_ID: ${{ inputs.client-id }} - PRIVATE_KEY: ${{ inputs.private-key }} run: | - if [ -n "$CLIENT_ID" ] && [ -n "$PRIVATE_KEY" ]; then + if [ -n "$CLIENT_ID" ]; then echo "has_app=true" >> "$GITHUB_OUTPUT" else echo "has_app=false" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0657224de7c3..beb9c41f6a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,6 @@ name: CI on: pull_request: - workflow_dispatch: push: branches: [main] @@ -43,6 +42,7 @@ jobs: timeout-minutes: 10 outputs: python: ${{ steps.classify.outputs.python }} + python_prod: ${{ steps.classify.outputs.python_prod }} frontend: ${{ steps.classify.outputs.frontend }} site: ${{ steps.classify.outputs.site }} scan: ${{ steps.classify.outputs.scan }} @@ -90,7 +90,19 @@ jobs: e2e-desktop: name: Desktop E2E needs: detect - if: needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true' + # python_prod (not python): the Playwright suite exercises the built app + # + `hermes serve` backend, which never import anything under tests/. + # Tests-only PRs (~17% of commits) skip this 5-minute job — the longest + # single job in the workflow — while still running the full pytest lanes. + # + # ⛔ TEMPORARILY DISABLED (Aug 2, 2026, Teknium) — the suite is red on + # every PR and on main itself since the Aug 1 night engines/npm churn + # (#76499 → #76562 → #76575): the mock-backend Electron window never + # gets a title, so boot/chat/setup/interim specs all fail identically + # regardless of the PR's diff (verified on #76573 and the docs-only + # #76582). Tracking issue: #76627 (assigned: Ari). To re-enable, + # delete the `false &&` below — nothing else changed. + if: ${{ false && (needs.detect.outputs.python_prod == 'true' || needs.detect.outputs.frontend == 'true') }} uses: ./.github/workflows/e2e-desktop.yml docs-site: @@ -116,6 +128,11 @@ jobs: needs: detect uses: ./.github/workflows/uv-lockfile-check.yml + infographic-check: + name: Check no committed infographics + needs: detect + uses: ./.github/workflows/infographic-check.yml + lockfile-diff: name: package-lock.json diff needs: detect @@ -133,8 +150,10 @@ jobs: needs: detect # Trusted main pushes run docker.yml directly so its container-publish # environment secrets never cross this reusable-workflow call. PR runs - # remain build/test-only and secret-free. - if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true' || needs.detect.outputs.docker_meta == 'true') + # remain build/test-only and secret-free. Gated on python_prod (not + # python): the image copies installed code, never tests/ — tests-only + # PRs skip the build. + if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.python_prod == 'true' || needs.detect.outputs.frontend == 'true' || needs.detect.outputs.docker_meta == 'true') uses: ./.github/workflows/docker.yml supply-chain: diff --git a/.github/workflows/contributor-check.yml b/.github/workflows/contributor-check.yml index 014e1e2ff93d..791e6305581d 100644 --- a/.github/workflows/contributor-check.yml +++ b/.github/workflows/contributor-check.yml @@ -67,6 +67,8 @@ jobs: echo -e "$MISSING" echo "" echo "Add a mapping file (do NOT edit AUTHOR_MAP in release.py):" + echo " python3 scripts/audit_pr_attribution.py --fix # auto-resolve + create files" + echo "or manually:" echo -e "$MISSING" | while read -r line; do email=$(echo "$line" | sed 's/^ *//' | cut -d' ' -f1) [ -z "$email" ] && continue @@ -78,7 +80,7 @@ jobs: # Emit review_status for unmapped emails DETAIL=$(echo -e "$MISSING" | sed '/^$/d; s/^ //') - HOW_TO_FIX=$'Add mappings to scripts/release.py AUTHOR_MAP:\n```\n"": "",\n```\nTo find the GitHub username for an email:\n```\ngh api \'search/users?q=EMAIL+in:email\' --jq \'.items[0].login\'\n```\n' + HOW_TO_FIX=$'Run from the PR branch:\n```\npython3 scripts/audit_pr_attribution.py --fix\ngit add contributors && git commit -m "chore: map contributor emails" && git push\n```\nOr map one email manually (do NOT edit AUTHOR_MAP in release.py):\n```\npython3 scripts/add_contributor.py \n```\nTo find the GitHub username for an email:\n```\ngh api \'search/users?q=EMAIL+in:email\' --jq \'.items[0].login\'\n```\n' REVIEW_STATUS=$(jq -nc \ --arg detail "$DETAIL" \ --arg how_to_fix "$HOW_TO_FIX" \ diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 3ac2c4741f89..588d7707ea76 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -65,10 +65,14 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 22 + node-version: 26 cache: npm cache-dependency-path: website/package-lock.json + - name: grab npm 12 + run: | + npm i -g npm@12 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5e5c19bdf3b6..7e47b1db693b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -53,7 +53,19 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Retry once on transient Docker Hub / buildkit pull failures + # (connection reset, auth token timeout, rate limiting). The action + # generates a unique builder name per invocation so the retry doesn't + # collide with the failed first attempt. A genuine persistent failure + # still fails the job — only the first attempt has continue-on-error. + # Refs: docker/setup-buildx-action#510 - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 # Build once, load into the local daemon for testing. Cached @@ -88,9 +100,16 @@ jobs: # --------------------------------------------------------------------- - name: Install uv (for docker tests) uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + # Pinned: unpinned setup-uv fetches a 'latest' manifest from + # raw.githubusercontent.com every job; transient fetch failures + # fail the job (2026-07-28 incident). Keep in sync with tests.yml. + version: "0.9.28" - name: Set up Python 3.11 (for docker tests) - run: uv python install 3.11 + uses: ./.github/actions/retry + with: + command: uv python install 3.11 - name: Install Python dependencies (for docker tests) # ``dev`` extra pulls in pytest, pytest-asyncio — @@ -141,7 +160,15 @@ jobs: - name: Checkout trusted source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Retry once on transient Docker Hub / buildkit pull failures. + # See build job for rationale; same pattern. - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Log in to Docker Hub @@ -203,7 +230,15 @@ jobs: pattern: digest-* merge-multiple: true + # Retry once on transient Docker Hub / buildkit pull failures. + # See build job for rationale; same pattern. - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Log in to Docker Hub diff --git a/.github/workflows/docs-site-checks.yml b/.github/workflows/docs-site-checks.yml index 41acf1790f48..cf775f89e032 100644 --- a/.github/workflows/docs-site-checks.yml +++ b/.github/workflows/docs-site-checks.yml @@ -15,10 +15,14 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 22 + node-version: 26 cache: npm cache-dependency-path: website/package-lock.json + - name: grab npm 12 + run: | + npm i -g npm@12 + - name: Install website dependencies uses: ./.github/actions/retry with: diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml index e9131c725224..8d0c037a377e 100644 --- a/.github/workflows/e2e-desktop.yml +++ b/.github/workflows/e2e-desktop.yml @@ -39,8 +39,13 @@ jobs: # ── Node ─────────────────────────────────────────────────────────── - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 22 + node-version: 26 cache: npm + + - name: grab npm 12 + run: | + npm i -g npm@12 + # Full npm ci (not --ignore-scripts): electron's postinstall # downloads the binary we launch, and node-pty's native build is # needed for the terminal pane. @@ -52,12 +57,21 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 with: + # Pin the uv version: unpinned, setup-uv resolves "latest" by + # fetching a manifest from raw.githubusercontent.com on EVERY job — + # a transient fetch failure fails the whole job (2026-07-28 slice-5 + # incident). Pinned, the binary downloads directly; no manifest hop. + version: '0.9.28' enable-cache: true cache-dependency-glob: | pyproject.toml uv.lock + - name: Set up Python 3.11 - run: uv python install 3.11 + uses: ./.github/actions/retry + with: + command: uv python install 3.11 + - name: Install Python dependencies uses: ./.github/actions/retry with: @@ -101,11 +115,11 @@ jobs: npx playwright test --reporter=list fi env: - CI: "true" + CI: 'true' # Ensure no real API keys leak into the test env. - OPENROUTER_API_KEY: "" - OPENAI_API_KEY: "" - NOUS_API_KEY: "" + OPENROUTER_API_KEY: '' + OPENAI_API_KEY: '' + NOUS_API_KEY: '' # ── Save updated baselines to cache (main only) ─────────────────── - name: Save updated baselines to cache diff --git a/.github/workflows/infographic-check.yml b/.github/workflows/infographic-check.yml new file mode 100644 index 000000000000..288f6f493a05 --- /dev/null +++ b/.github/workflows/infographic-check.yml @@ -0,0 +1,78 @@ +name: Infographic Check + +# Rejects PRs that commit PR-infographic images into the repo. +# +# PR infographics are rendered to an image-provider URL (fal.media) and +# embedded in the PR *description*. The PR body is the archive; the binary +# never belongs in git history. +# +# This has now leaked twice. PR #48261 removed the first batch, PR #54564 +# removed a second batch and added `infographic/` to `.gitignore` — but +# `.gitignore` only stops *accidental* `git add`. It does nothing against +# `git add -f`, and it does nothing for a path that does not literally match +# the ignore pattern. Nine more PNGs (~14MB) were committed in the four +# weeks AFTER that rule landed, plus PR #70552 caught an `infograficos/` +# spelling that sidestepped the pattern entirely. +# +# A passive ignore rule cannot enforce a policy. This check can. + +on: + workflow_call: + outputs: + review_status: + description: "JSON array of review_status objects for the synthesizer." + value: ${{ jobs.check-no-committed-infographics.outputs.review_status }} + +permissions: + contents: read + +jobs: + check-no-committed-infographics: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + review_status: ${{ steps.infographic-check.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - id: infographic-check + name: Reject committed PR-infographic images + run: | + # Match on the IMAGE, not on a directory name. Keying this to + # `infographic/` is what let `infograficos/` through in #70552 — + # any localized or typo'd directory would sidestep it again. + # Instead: find tracked raster images whose path contains an + # infographic-ish segment, in any spelling, at any depth. + # + # `docs/assets` and `website/` legitimately hold product imagery + # and are excluded; those are referenced from shipped docs pages. + OFFENDERS=$(git ls-files -z \ + | tr '\0' '\n' \ + | grep -iE '(^|/)(infograph|infograf)[^/]*/' \ + | grep -iE '\.(png|jpe?g|webp|gif)$' \ + || true) + + if [ -n "$OFFENDERS" ]; then + COUNT=$(printf '%s\n' "$OFFENDERS" | wc -l | tr -d ' ') + STATUS='[{"source":"committed infographics","results":[{"kind":"action_required","title":"PR infographic committed to the repo","summary":"Infographic images belong in the PR description, never in git.","detail":"","how_to_fix":"Untrack the image and reference the provider URL from the PR body instead:\n```\ngit rm --cached \n```\nThen put it in the PR description:\n```\n## Infographic\n\n![slug](https://)\n```\n"}]}]' + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" + echo "" + echo "::error::${COUNT} PR-infographic image(s) are tracked in git." + echo "" + printf '%s\n' "$OFFENDERS" | sed 's/^/ /' + echo "" + echo "PR infographics are rendered to an image-provider URL and" + echo "embedded in the PR DESCRIPTION. The PR body is the archive —" + echo "the binary never enters git history." + echo "" + echo "This rule has been re-established twice already (#48261," + echo "#54564) and leaked both times, because .gitignore cannot stop" + echo "'git add -f' or a differently-spelled directory (#70552)." + echo "" + echo "To fix:" + echo " git rm --cached # keeps your local copy" + echo " # then embed the provider URL in the PR description" + exit 1 + fi + echo "::notice::No committed PR-infographic images." + echo "review_status=[]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/js-autofix.yml b/.github/workflows/js-autofix.yml index 232b9c3a7472..d0fa3513e86c 100644 --- a/.github/workflows/js-autofix.yml +++ b/.github/workflows/js-autofix.yml @@ -67,9 +67,13 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 22 + node-version: 26 cache: npm + - name: grab npm 12 + run: | + npm i -g npm@12 + # --ignore-scripts: eslint only needs TS sources + eslint packages. - uses: ./.github/actions/retry with: @@ -128,9 +132,6 @@ jobs: pull-requests: write # needed for PR creation + auto-merge steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - # The App token below must own the push so its commit triggers CI. - persist-credentials: false - name: Get GitHub App token id: app-token @@ -152,7 +153,6 @@ jobs: - name: Apply patch and push to bot branch env: BOT_BRANCH: bot/js-autofix - GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | set -euo pipefail @@ -178,9 +178,6 @@ jobs: # bot/js-autofix is a bot-only branch that gets rewritten each run. # If the branch was deleted after a previous PR merge, this # recreates it. - # Configure Git's credential helper from GH_TOKEN. The App token - # stays in the step environment and never enters the remote URL. - gh auth setup-git git push --force origin HEAD:"$BOT_BRANCH" - name: Create/update PR and enable auto-merge diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 25786b1c95b6..9119e0c7a9b9 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -15,8 +15,13 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 22 + node-version: 26 cache: npm + + - name: grab npm 12 + run: | + npm i -g npm@12 + - uses: ./.github/actions/retry with: command: npm ci --ignore-scripts @@ -61,8 +66,13 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 22 + node-version: 26 cache: npm + + - name: grab npm 12 + run: | + npm i -g npm@12 + - uses: ./.github/actions/retry with: command: npm ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 670b6f2a44a2..7a49a6220395 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,6 +40,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + # Pinned: unpinned setup-uv fetches a 'latest' manifest from + # raw.githubusercontent.com every job; transient fetch failures + # fail the job (2026-07-28 incident). Keep in sync with tests.yml. + version: "0.9.28" - name: Install ruff + ty uses: ./.github/actions/retry @@ -129,6 +134,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + # Pinned: unpinned setup-uv fetches a 'latest' manifest from + # raw.githubusercontent.com every job; transient fetch failures + # fail the job (2026-07-28 incident). Keep in sync with tests.yml. + version: "0.9.28" - name: Install ruff uses: ./.github/actions/retry @@ -153,10 +163,18 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v5 + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + # Pinned: unpinned setup-uv fetches a 'latest' manifest from + # raw.githubusercontent.com every job; transient fetch failures + # fail the job (2026-07-28 incident). Keep in sync with tests.yml. + version: "0.9.28" + + - name: Set up Python 3.11 + uses: ./.github/actions/retry with: - python-version: "3.11" + command: uv python install 3.11 - name: Run footgun checker run: python scripts/check-windows-footguns.py --all diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml index 455ede33dd56..c3aaa50a7b5a 100644 --- a/.github/workflows/osv-scanner.yml +++ b/.github/workflows/osv-scanner.yml @@ -43,11 +43,13 @@ jobs: uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 with: # Scan explicit lockfiles rather than recursing, so we only look at - # the three sources of truth and skip vendored / test / worktree dirs. + # the five sources of truth and skip vendored / test / worktree dirs. scan-args: |- --lockfile=uv.lock --lockfile=package-lock.json --lockfile=website/package-lock.json + --lockfile=plugins/platforms/photon/sidecar/package-lock.json + --lockfile=scripts/whatsapp-bridge/package-lock.json # The upstream reusable workflow uploads this exact file under its # fixed artifact name, which the wrapper downloads below. results-file-name: osv-results.sarif diff --git a/.github/workflows/supply-chain-audit.yml b/.github/workflows/supply-chain-audit.yml index 7ca09d6c02d2..cca61e03a452 100644 --- a/.github/workflows/supply-chain-audit.yml +++ b/.github/workflows/supply-chain-audit.yml @@ -19,9 +19,9 @@ name: Supply Chain Audit # review_status — JSON array of status objects consumed by the review # comment assembler (scripts/ci/assemble_review_comment.py). # critical_findings — "true" when the narrow critical-pattern scan found -# something. The scanner publishes the exact evidence; -# the review-label gate consumes this boolean and owns -# approval, so adding ``ci-reviewed`` can heal the run. +# something. The review-label gate consumes this and +# owns the action-required result, so adding +# ``ci-reviewed`` can heal the run on rerun. on: workflow_call: @@ -162,16 +162,19 @@ jobs: id: emit-status if: always() env: - CI_REVIEWED: ${{ contains(github.event.pull_request.labels.*.name, 'ci-reviewed') }} + FOUND: ${{ steps.scan.outputs.found }} run: | - args=( - --supply-chain-findings-file /tmp/findings.md - --output "$GITHUB_OUTPUT" - ) - if [ "$CI_REVIEWED" = "true" ]; then - args+=(--label-present) - fi - python3 scripts/ci/emit_review_status.py "${args[@]}" + python3 - <<'PYEOF' + import json, os + + # The review-label gate renders and blocks critical findings. Keep + # this scan a fact-finder so adding ci-reviewed can rerun the gate + # without requiring the scanner itself to fail again. + status = [] + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"review_status={json.dumps(status)}\n") + PYEOF dep-bounds: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cdae2e037a59..3888e8d44f9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,6 +74,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 with: + # Pin the uv version: unpinned, setup-uv resolves "latest" by + # fetching a manifest from raw.githubusercontent.com on EVERY job — + # a transient fetch failure fails the whole job (2026-07-28 slice-5 + # incident). Pinned, the binary downloads directly; no manifest hop. + version: "0.9.28" # Persist uv's download/wheel cache (~/.cache/uv) across runs. # Keyed on the dependency manifests, so the cache is reused until # pyproject.toml or uv.lock changes. `uv sync` still runs every @@ -85,16 +90,27 @@ jobs: uv.lock - name: Set up Python 3.11 - run: uv python install 3.11 + uses: ./.github/actions/retry + with: + command: uv python install 3.11 - name: Install dependencies # `uv sync --locked` installs the exact pinned set from uv.lock (and # fails if the lock is out of sync with pyproject.toml), giving a # reproducible env. It also creates .venv itself, so no separate # `uv venv` step is needed. + # + # The trailing extras beyond all/dev are the lazy-install features + # (tools/lazy_deps.py) that tests exercise for real: provider.anthropic, + # stt/tts.mistral, image.fal, terminal.modal, terminal.daytona, + # memory.hindsight, search.parallel. The hermetic test env forbids + # mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in + # tests/conftest.py), so the SDKs those tests need must be in the + # venv up front — resolved from uv.lock like everything else, which + # also honors the exact supply-chain pins these extras carry. uses: ./.github/actions/retry with: - command: uv sync --locked --python 3.11 --extra all --extra dev + command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web - name: Minimize uv cache # Optimized for CI: prunes pre-built wheels that are cheap to @@ -188,6 +204,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 with: + # Pin the uv version: unpinned, setup-uv resolves "latest" by + # fetching a manifest from raw.githubusercontent.com on EVERY job — + # a transient fetch failure fails the whole job (2026-07-28 slice-5 + # incident). Pinned, the binary downloads directly; no manifest hop. + version: "0.9.28" # Persist uv's download/wheel cache (~/.cache/uv) across runs. # Keyed on the dependency manifests, so the cache is reused until # pyproject.toml or uv.lock changes. `uv sync` still runs every @@ -206,9 +227,14 @@ jobs: # fails if the lock is out of sync with pyproject.toml), giving a # reproducible env. It also creates .venv itself, so no separate # `uv venv` step is needed. + # + # Same extras as the test job's sync above: the hermetic test env + # forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in + # tests/conftest.py), so lazy-install SDKs exercised by tests must be + # in the venv up front. uses: ./.github/actions/retry with: - command: uv sync --locked --python 3.11 --extra all --extra dev + command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web - name: Minimize uv cache # Optimized for CI: prunes pre-built wheels that are cheap to diff --git a/.github/workflows/uv-lockfile-check.yml b/.github/workflows/uv-lockfile-check.yml index aff4f0eb8cbd..e0ba3ea7d508 100644 --- a/.github/workflows/uv-lockfile-check.yml +++ b/.github/workflows/uv-lockfile-check.yml @@ -70,6 +70,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + # Pinned: unpinned setup-uv fetches a 'latest' manifest from + # raw.githubusercontent.com every job; transient fetch failures + # fail the job (2026-07-28 incident). Keep in sync with tests.yml. + version: "0.9.28" # `uv lock --check` re-resolves the project from pyproject.toml and # compares the result to uv.lock, exiting non-zero if they disagree. diff --git a/.gitignore b/.gitignore index 23c0a8555ad1..55a2ab8aade6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .DS_Store /venv/ /venv.old/ +/venv.stale.runtime-*/ +/.hermes-runtime/ /_pycache/ *.pyc* __pycache__/ @@ -29,7 +31,6 @@ __pycache__/model_tools.cpython-310.pyc __pycache__/web_tools.cpython-310.pyc logs/ data/ -cache/ .pytest_cache/ test_durations.json .pytest-cache/ @@ -142,11 +143,6 @@ docs/superpowers/* # treat it as a local edit and autostash it on every run (#38529). .hermes-bootstrap-complete -# Install-method stamp written next to managed code by install.sh/Dockerfile. -# It describes the local runtime install, not the source tree, and must not keep -# every bootstrapped checkout dirty. -.install_method - # Persistent dev sandbox dir (scripts/dev-sandbox.sh --persistent) .hermes-sandbox/ @@ -156,6 +152,11 @@ docs/superpowers/* .update-incomplete .update-incomplete.lock +# Checkout fingerprint the __pycache__ tree was last validated against +# (launch-time stale-bytecode sweep). Runtime state, never a code change. +.bytecode-fingerprint +.bytecode-fingerprint.tmp + # Installer-written method stamp in the managed checkout root (scripts/install.sh). # Runtime metadata only — never a code change. Ignore so `git status` stays clean # and `hermes update`'s untracked autostash does not treat it as a local edit (#66189 / #54855). @@ -179,5 +180,17 @@ apps/desktop/demo/ # image-provider (fal.media) URL — they are NEVER committed to the repo. The # PR body is the archive. See the hermes-agent-dev skill's # pr-infographic-workflow reference (storage rule + lapse #8 / #COMMIT-1). +# +# Spelling variants are listed because a single `infographic/` pattern was +# sidestepped by an `infograficos/` directory (#70552). .gitignore is only +# the first line of defence and cannot stop `git add -f` at all — the +# infographic-check CI job is what actually enforces this. infographic/ +infographics/ +infograficos/ +infografico/ native/fts5_cjk/*.so +# Runtime marker written by hermes update when a lazy dependency refresh is +# interrupted; consumed by launch-time recovery. Never commit it (was tracked +# by accident via 3a69e34702, removed in the #72002 salvage). +.lazy-refresh-incomplete diff --git a/.npmrc b/.npmrc new file mode 100644 index 000000000000..d25bd185bbe1 --- /dev/null +++ b/.npmrc @@ -0,0 +1,51 @@ +# needed to prevent bad npm that has min-release-age but not exclude +engine-strict=true + +min-release-age=14 +# allow assistant-ui packages & a couple specific deps since they update a LOT. +# remove this when we stabilize (or we haven't updated in 2 wks) +min-release-age-exclude[]=@assistant-ui/* +min-release-age-exclude[]=assistant-cloud +min-release-age-exclude[]=assistant-stream +min-release-age-exclude[]=@radix-ui/* +min-release-age-exclude[]=radix-ui +min-release-age-exclude[]=safe-content-frame + +# react-router 8.3.0 includes fixes for vulns. remove this when 8.3.0 is > 2wks old. +min-release-age-exclude[]=react-router + +# eslint 10.8.0 includes fixes for vulns. remove this when 10.8.0 is > 2wks old. +min-release-age-exclude[]=eslint +min-release-age-exclude[]=@eslint/* + +# tar 7.5.21 includes fixes for vulns. remove this when 7.5.21 is > 2wks old +min-release-age-exclude[]=tar + +# concurrently 10.0.4 includes fixes for vulns. remove this when 10.0.4 is > 2wks old +min-release-age-exclude[]=concurrently + +# fast-uri 3.1.4 includes fixes for vulns. remove this when 3.1.4 is > 2wks old +min-release-age-exclude[]=fast-uri + +# minimatch 10.2.6 includes fixes for vulns. remove this when 10.2.6 is > 2wks old +min-release-age-exclude[]=minimatch + + +# brace-expansion 5.0.8 includes fixes for vulns. remove this when 5.0.8 is > 2wks old +min-release-age-exclude[]=brace-expansion + +# vite 8.2.0 is the first release depending on rolldown >= 1.2.1, which fixes +# a rolldown panic that breaks `npm run build` in apps/desktop +# (rolldown/rolldown#10337 — a regression in 1.1.5, the version vite 8.1.5 +# pins as ~1.1.5). @oxc-project/types is here because rolldown 1.2.1 pins it +# as `=0.142.0` — an exact pin, so no older release satisfies it and the age +# gate would fail the whole install with ETARGET. +# remove these once vite 8.2.0 is > 2wks old. +min-release-age-exclude[]=vite +min-release-age-exclude[]=rolldown +min-release-age-exclude[]=@rolldown/* +min-release-age-exclude[]=@oxc-project/types + +# ink needs +min-release-age-exclude[]=lightningcss +min-release-age-exclude[]=postcss diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000000..6f4247a6255c --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +26 diff --git a/.plans/message-reactions.md b/.plans/message-reactions.md new file mode 100644 index 000000000000..377d29b1763f --- /dev/null +++ b/.plans/message-reactions.md @@ -0,0 +1,146 @@ +# Message reactions (desktop tapbacks) + +Two-way emoji reactions on individual messages in the desktop transcript: the +user reacts to any message, the agent reacts to a user message, and both sides +read the other's reactions as conversational signal. + +## What already exists + +Hermes already models reactions on the **platform** side — the desktop is the +only surface without them. + +| Surface | Reaction support | Where | +|---|---|---| +| Agent → platform message | `send_message(action="react"/"unreact")` | `tools/send_message_tool.py:266` `_handle_react()` | +| Photon / iMessage | tapbacks in + out, routed only for messages we sent | `plugins/platforms/photon/adapter.py:1240-1283` | +| Telegram | `setMessageReaction`, config-gated | `plugins/platforms/telegram/adapter.py:9669+` | +| Slack / Matrix / Feishu / Discord | inbound reaction events → hooks | `gateway/run.py:4688` `_handle_reaction_event()` → `HookRegistry.emit("reaction:added")` | +| Adapter contract | `add_reaction()` / `remove_reaction()` coroutines, `set_reaction_handler()` | `gateway/platforms/base.py:3330` | +| Core "affection" detector | regex on user text → `vibe`, drives CLI pet / TUI heart / desktop hearts | `agent/reactions.py`, `agent/turn_context.py:592-604` | + +Two things follow from that table: + +1. **The agent-facing verb already exists.** `send_message(action="react")` is + the established shape. A desktop reaction should extend that tool, not add a + new core tool — every new tool ships on every API call (AGENTS.md footprint + ladder). +2. **The inbound convention already exists.** Photon turns a tapback into a + normal message event with `reply_to_message_id` + `reply_to_is_own_message`, + and the gateway prefixes `[Replying to your previous message: "…"]` + (`gateway/run.py:13125-13132`). Desktop reactions should read the same way to + the model. + +Nothing exists on the desktop side: `grep -ri reaction` across `apps/desktop` +finds only the pet-overlay hearts. + +## Prior art + +**iOS Tapback** ([Apple](https://support.apple.com/guide/iphone/react-with-tapbacks-iph018d3c336/ios)): +double-tap or touch-and-hold a message → floating pill above the bubble with +heart / thumbs-up / thumbs-down / haha / ‼️ / ❓, swipe left for suggested emoji +and stickers, or tap the emoji button for the full keyboard. **One tapback per +message per person** — tapping the same one again removes it, tapping a +different one replaces it. Multiple people's tapbacks stack on the badge. + +**Platform data models** converge on the same shape: + +| Platform | Model | Add / remove | +|---|---|---| +| Slack | `{name, count, users[]}` | [`reactions.add`](https://docs.slack.dev/reference/methods/reactions.add) / `reactions.remove`, emits `reaction_added` | +| Discord | `{emoji, count, me}` on the message object | `PUT`/`DELETE .../reactions/{emoji}/@me` | +| Telegram | `reaction: [{type:"emoji", emoji:"👍"}]` — replaces the whole set | `setMessageReaction`, `is_big` for the big animation | + +Telegram's "set the whole array" is the closest match to iOS semantics and the +simplest thing to persist. + +**assistant-ui has no reaction primitive.** `@assistant-ui/react` 0.14.24 (MIT, +vendored at `apps/desktop/node_modules`): zero hits for "reaction" in `core/src`, +`react/src`, `dist/`, or the 2.2 MB `llms-full.txt` docs dump. What exists is a +hard-coded binary `FeedbackAdapter` (`"positive" | "negative"`, +`core/src/adapters/feedback.ts`) that throws when unconfigured and only writes +back onto assistant messages. Not usable for emoji, not usable on user messages. + +**But `metadata.custom` is the supported extension channel** and this repo +already uses it: `ThreadUserMessage`/`ThreadAssistantMessage`/`ThreadSystemMessage` +all carry `metadata.custom: Record` (`core/src/types/message.ts:319-366`), +and `chat-runtime.ts:397` already ships `custom: { attachmentRefs }` through it. + +**Emoji picker survey** (npm week of 2026-07-22, sizes measured from the +published ESM entry): + +| Library | License | Weekly DL | gzip | Headless | Latest | +|---|---|---|---|---|---| +| **frimousse** | MIT | 573k | **8.5 kB** | ✅ fully unstyled, composable parts | 0.3.0 · 2025-07-15 | +| emoji-picker-react | MIT | 1.31M | 87 kB | ❌ own CSS-in-JS (flairup) | 4.19.1 · 2026-04-27 | +| emoji-mart | MIT | 2.22M | ~120 kB w/ data | ❌ Preact + shadow styling | 5.6.0 · **2024-04-25**, 217 open issues | +| emoji-picker-element | Apache-2.0 | 183k | — | ❌ Web Component / Shadow DOM | 1.29.1 · 2026-03-01 | + +No picker is currently a dependency (only `emoji-regex`, transitive). Already +paid for and reusable: `radix-ui` (Popover), `motion`, `@tanstack/react-virtual`, +Tailwind v4. + +## Recommendation + +**Hand-roll the tapback pill; add frimousse only behind the "+".** Six fixed +emoji in a pill is ~40 lines of JSX against existing tokens — pulling 87 kB of +`emoji-picker-react` to render six buttons, plus a CSS engine that fights +`DESIGN.md`, is backwards. frimousse is headless, dependency-free, 10× smaller, +and exposes `emojibaseUrl` so the data can be bundled as a Vite asset instead of +hitting jsDelivr (Electron must work offline). + +### Data model + +One reaction per author per message, Telegram-style whole-set replacement: + +```ts +type MessageReaction = { emoji: string; author: 'user' | 'agent'; at: number } +``` + +Persisted in the existing `messages.display_metadata` JSON column +(`hermes_state_common.py:215`) — no new table. It already survives insert, +compaction, and every read projection, and +`set_latest_matching_message_display_kind()` (`hermes_state.py:5292`) is the +precedent for stamping metadata onto an already-persisted row. + +### Model context + +Reactions must reach the model **without breaking prompt caching**. The +`api_messages` build loop strips `display_metadata` from every outgoing copy +(`agent/conversation_loop.py:1443-1446`) precisely so display state never +becomes a provider field. Two candidate paths: + +| Path | Cache impact | Notes | +|---|---|---| +| Rewrite the reacted-to message's content to carry the annotation | **Breaks the cached prefix** — mutates past context | Rejected. AGENTS.md: prompt caching is sacred. | +| Deliver the reaction as the *next* turn's leading annotation, mirroring photon | Prefix untouched; only the new turn carries it | Matches `[Replying to your previous message: "…"]` (`gateway/run.py:13125`), which the agent already understands | + +The second is the same trick the platform adapters already use, so the model +sees a familiar shape and no existing conversation is rewritten. + +### Attach points + +| Concern | File | Lines | +|---|---|---| +| Assistant hover bar | `apps/desktop/src/components/assistant-ui/thread/assistant-message.tsx` | 134–175 | +| User hover cluster | `apps/desktop/src/components/assistant-ui/thread/user-message.tsx` | 296–336 | +| Callback threading (ref caveat 79–99) | `apps/desktop/src/components/assistant-ui/thread/index.tsx` | 109–133 | +| `metadata.custom` → runtime | `apps/desktop/src/lib/chat-runtime.ts` | 384–432 | +| RPC client ↔ server pattern | `sidebar/session-actions-menu.tsx:62-89` ↔ `tui_gateway/server.py:8322` | — | +| Persistence | `hermes_state_common.py:192-216`, `hermes_state.py:5292-5324` | — | +| Prompt injection / strip | `agent/conversation_loop.py` | 1430–1529 | + +### Known gaps to solve first + +- **No durable message id crosses the gateway RPC path.** `_history_to_messages()` + (`tui_gateway/server.py:6545`) builds `{"role", "text"}` and drops the id. The + REST path carries `messages.id` incidentally via `SELECT *` but TS + `SessionMessage` (`types/hermes.ts:513-533`) doesn't declare it. Renderer ids + are ephemeral and change shape between rehydrated (`--`), live + (`assistant-`), and optimistic (`user--`) messages. A reaction + needs a stable key — this is the first thing to fix. +- **WeakMap identity cache** in `apps/desktop/src/app/chat/runtime-repository.ts:26-66` + keys normalized `ThreadMessage` by `ChatMessage` identity. A reaction change + must produce a **new** `ChatMessage` object or the UI renders stale. +- **Rewind rewrites rows** (`replace_messages`), so anything keyed by row id + needs cascade handling — an argument for keeping reactions in + `display_metadata` on the row itself rather than a side table. diff --git a/.python-version b/.python-version new file mode 100644 index 000000000000..2c0733315e41 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/AGENTS.md b/AGENTS.md index cb53e95eb0b9..70fc9bc8d647 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -325,7 +325,7 @@ class AIAgent: provider: str = None, api_mode: str = None, # "chat_completions" | "codex_responses" | ... model: str = "", # empty → resolved from config/provider later - max_iterations: int = 90, # tool-calling iterations (shared with subagents) + max_iterations: int = 500, # tool-calling iterations (shared with subagents) enabled_toolsets: list = None, disabled_toolsets: list = None, quiet_mode: bool = False, @@ -1284,14 +1284,15 @@ def profile_env(tmp_path, monkeypatch): ### Python **ALWAYS use `scripts/run_tests.sh`** — do not call `pytest` directly. The script enforces hermetic environment parity with CI (unset credential vars, TZ=UTC, LANG=C.UTF-8, -`-n auto` xdist workers, in-tree subprocess-isolation plugin). Direct `pytest` +per-file subprocess isolation via `scripts/run_tests_parallel.py` — no xdist, +worker count auto-scaled from CPU count). Direct `pytest` on a 16+ core developer machine with API keys set diverges from CI in ways that have caused multiple "works locally, fails in CI" incidents (and the reverse). ```bash scripts/run_tests.sh # full suite, CI-parity scripts/run_tests.sh tests/gateway/ # one directory -scripts/run_tests.sh tests/agent/test_foo.py::test_x # one test +scripts/run_tests.sh tests/agent/test_foo.py -k test_x # one test (file + -k; the runner is file-granular) scripts/run_tests.sh -v --tb=long # pass-through pytest flags ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46581d820037..4fbf5b5a3da6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -201,7 +201,8 @@ ln -sf "$(pwd)/venv/bin/hermes" ~/.local/bin/hermes ### Run tests ```bash -# Preferred — matches CI (hermetic env, 4 xdist workers); see AGENTS.md +# Preferred — matches CI (hermetic `env -i`, per-file subprocess isolation +# via run_tests_parallel.py, worker count auto-scaled); see AGENTS.md scripts/run_tests.sh # Alternative (activate the venv first). The wrapper is still recommended @@ -848,7 +849,7 @@ that touches the OS, assume *any* platform can hit your code path. Tests that use POSIX-only syscalls need a skip marker. Common ones: - Symlinks → `@pytest.mark.skipif(sys.platform == "win32", ...)` - `0o600` file modes → `@pytest.mark.skipif(sys.platform.startswith("win"), ...)` -- `signal.SIGALRM` → Unix-only (see `tests/conftest.py::_enforce_test_timeout`) +- `signal.SIGALRM` → Unix-only (per-test timeouts no longer use it directly; see the win32 timeout-method shim in `tests/conftest.py::pytest_configure`) - `os.setsid` / `os.fork` → Unix-only - Live Winsock / Windows-specific regression tests → `@pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific regression")` diff --git a/Dockerfile b/Dockerfile index 388056faacde..2de6192715ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,54 @@ +# Debian 13 still ships SQLite 3.46.1, which contains the upstream WAL-reset +# corruption bug. Build a pinned shared library for the runtime image instead +# of relying on a distro backport that trixie does not currently provide. +# See #70480 and https://sqlite.org/wal.html#walresetbug. +FROM debian:13.4 AS sqlite_build +ARG SQLITE_AUTOCONF_VERSION=3530400 +ARG SQLITE_SHA256=0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c +RUN apt-get -o Acquire::Retries=3 update && \ + apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + build-essential ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* && \ + (curl -fsSL --retry 1 --retry-all-errors --connect-timeout 15 --max-time 60 \ + -o /tmp/sqlite.tar.gz \ + "https://sqlite.org/2026/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}.tar.gz" || \ + curl -fsSL --retry 3 --retry-all-errors --connect-timeout 15 --max-time 120 \ + -o /tmp/sqlite.tar.gz \ + "https://sources.buildroot.net/sqlite/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}.tar.gz") && \ + printf '%s %s\n' "${SQLITE_SHA256}" /tmp/sqlite.tar.gz > /tmp/sqlite.sha256 && \ + sha256sum -c /tmp/sqlite.sha256 && \ + tar -xzf /tmp/sqlite.tar.gz -C /tmp && \ + cd "/tmp/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}" && \ + CFLAGS="-O2 \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_FTS3_PARENTHESIS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_COLUMN_METADATA \ + -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_SECURE_DELETE \ + -DSQLITE_THREADSAFE=1 \ + -DSQLITE_MAX_VARIABLE_NUMBER=250000" \ + ./configure --prefix=/opt/sqlite-fixed --disable-static && \ + make -j"$(nproc)" && \ + make install + FROM ghcr.io/astral-sh/uv:0.11.6-python3.13-trixie@sha256:b3c543b6c4f23a5f2df22866bd7857e5d304b67a564f4feab6ac22044dde719b AS uv_source -# Node 22 LTS source stage. Debian trixie's bundled nodejs is pinned to 20.x -# which reached EOL in April 2026 — we copy node + npm + corepack from the -# upstream node:22 image instead so we can stay on a supported LTS without -# waiting for Debian 14 (forky, ~mid-2027). Bookworm-based slim image used -# so the produced binary links against glibc 2.36, which runs cleanly on -# our Debian 13 (trixie, glibc 2.41) runtime. Bumping to a new Node major -# is a one-line ARG change; see #4977. -FROM node:22-bookworm-slim@sha256:7af03b14a13c8cdd38e45058fd957bf00a72bbe17feac43b1c15a689c029c732 AS node_source +# Node 26 source stage. Debian trixie's bundled nodejs is pinned to 20.x +# which reached EOL in April 2026 — we copy node + npm from the upstream +# node:26 image instead (Hermes pins its toolchain to Node 26 everywhere). +# Bookworm-based slim image used so the produced binary links +# against glibc 2.36, which runs cleanly on our Debian 13 (trixie, glibc +# 2.41) runtime. Bumping to a new Node major is a one-line ARG change; see +# #4977. +FROM node:26-bookworm-slim@sha256:9e6f9357d371591e32ab6f2d8a26d63bdd0d17c29eee3f4f3e7e454d9634bf73 AS node_source FROM debian:13.4 # Disable Python stdout buffering to ensure logs are printed immediately. @@ -28,9 +70,26 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/opt/hermes/.playwright # hermes process, the dashboard, and per-profile gateways. RUN apt-get -o Acquire::Retries=3 update && \ apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ - ca-certificates curl iputils-ping python3 python-is-python3 ripgrep ffmpeg gcc g++ make cmake python3-dev python3-venv libffi-dev libolm-dev procps git openssh-client docker-cli xz-utils && \ + ca-certificates curl iputils-ping python3 python-is-python3 ripgrep ffmpeg gcc g++ make cmake python3-dev python3-venv libffi-dev libolm-dev libatomic1 procps git openssh-client docker-cli xz-utils && \ rm -rf /var/lib/apt/lists/* +# Prefer the fixed SQLite over Debian's vulnerable libsqlite3.so.0. Keep the +# public library name stable so both the system interpreter and the uv-created +# venv resolve the replacement without changing Python import paths. +COPY --from=sqlite_build /opt/sqlite-fixed/lib/libsqlite3.so.3.53.4 /usr/local/lib/ +RUN ln -sf libsqlite3.so.3.53.4 /usr/local/lib/libsqlite3.so.0 && \ + ln -sf libsqlite3.so.3.53.4 /usr/local/lib/libsqlite3.so && \ + printf '/usr/local/lib\n' > /etc/ld.so.conf.d/000-sqlite-fixed.conf && \ + ldconfig && \ + python3 -c "import sqlite3, sys; \ +v = sqlite3.sqlite_version_info; \ +sys.exit(f'linked SQLite {sqlite3.sqlite_version} still has the WAL-reset bug') if v < (3, 51, 3) else None; \ +db = sqlite3.connect(':memory:'); \ +db.execute(\"CREATE VIRTUAL TABLE docs USING fts5(content, tokenize='trigram')\"); \ +db.execute(\"INSERT INTO docs VALUES ('hermes')\"); \ +sys.exit('SQLite FTS5 trigram self-test failed') if db.execute(\"SELECT count(*) FROM docs WHERE docs MATCH 'erm'\").fetchone()[0] != 1 else None; \ +db.close()" + # ---------- s6-overlay install ---------- # s6-overlay provides supervision for the main hermes process, the dashboard, # and per-profile gateways. /init becomes PID 1 below — see ENTRYPOINT. @@ -92,17 +151,20 @@ RUN useradd -u 10000 -m -d /opt/data hermes COPY --chmod=0755 --from=uv_source /usr/local/bin/uv /usr/local/bin/uvx /usr/local/bin/ -# Node 22 LTS: copy the node binary plus the bundled npm + corepack JS -# installs from the upstream image. npm and npx are recreated as symlinks -# because they're symlinks in the source image (and need to live on PATH). +# Node 26: copy the node binary plus the bundled npm JS install from the +# upstream image. npm and npx are recreated as symlinks because they're +# symlinks in the source image (and need to live on PATH). +# +# No corepack: Node unbundled it upstream, so node:26 ships only npm in +# /usr/local/lib/node_modules. Nothing here needs it — no package.json +# declares a `packageManager`, and no build step shells out to yarn or pnpm. +# # See node_source stage at the top of the file for the version-bump # rationale (#4977). COPY --chmod=0755 --from=node_source /usr/local/bin/node /usr/local/bin/ COPY --from=node_source /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm -COPY --from=node_source /usr/local/lib/node_modules/corepack /usr/local/lib/node_modules/corepack RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ - ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx && \ - ln -sf /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack + ln -sf /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx WORKDIR /opt/hermes @@ -141,6 +203,22 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ done && \ npm cache clean --force +# ---------- Photon iMessage sidecar deps (baked, NS-606) ---------- +# The photon plugin's Node sidecar needs its own node_modules +# (spectrum-ts). The install tree is immutable at runtime, so a lazy +# `npm ci` on first connect would hit EROFS — bake the deps here instead +# (deterministic installs, NS-559). The patch script is copied alongside +# the manifests because package.json's postinstall runs it, which also +# means the spectrum-ts patch is applied at build time. Layer-cached: +# only re-runs when the sidecar manifests/patch change. +COPY plugins/platforms/photon/sidecar/package.json \ + plugins/platforms/photon/sidecar/package-lock.json \ + plugins/platforms/photon/sidecar/patch-spectrum-mixed-attachments.mjs \ + plugins/platforms/photon/sidecar/ +RUN cd plugins/platforms/photon/sidecar && \ + npm ci --no-audit --fetch-retries=5 && \ + npm cache clean --force + # ---------- Layer-cached Python dependency install ---------- # Copy only pyproject.toml + uv.lock so the Python dep resolve + wheel # download + native-extension compile layer is cached unless those inputs @@ -152,7 +230,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # frontend stats the readme path during dep resolution, so we `touch` an # empty placeholder — the real README is restored by `COPY . .` below. # -# `uv sync --frozen --no-install-project --extra all --extra messaging` +# `uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp` # installs the deps reachable through the composite `[all]` extra # (handpicked set intended for the production image — excludes `[dev]`), # plus gateway messaging adapters that should work in the published image @@ -165,6 +243,10 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # so Docker users can use these providers without requiring runtime # lazy-install access to PyPI (often blocked in containerized envs). # +# The [otlp] extra contains the SDK/exporter imported by Hermes when Gateway +# Health export is enabled. Collector and observability-backend dependencies +# remain external and are not part of the Hermes production image. +# # The hindsight memory provider's client (hindsight-client) is baked in # for the same reason: it lazy-installs into /opt/hermes/.venv at first # use, which lives inside the (immutable) image layer rather than the @@ -182,7 +264,7 @@ RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ # The editable link is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md -RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix +RUN uv sync --frozen --no-install-project --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock --extra azure-identity --extra hindsight --extra matrix # ---------- Frontend build (cached independently from Python source) ---------- # Copy only the frontend source trees first so that Python-only changes don't @@ -321,6 +403,8 @@ ENV HERMES_LAZY_INSTALL_TARGET=/opt/data/lazy-packages # Recursion is impossible because the shim exec's the venv binary by # absolute path (/opt/hermes/.venv/bin/hermes). See the shim source for # the opt-out env var (HERMES_DOCKER_EXEC_AS_ROOT=1). +COPY --chmod=0755 docker/hermes-exec-shim.sh /opt/hermes/bin/hermes +COPY --chmod=0755 docker/entrypoint-dispatch.sh /opt/hermes/docker/entrypoint-dispatch.sh # Pre-s6 entrypoint.sh did `source .venv/bin/activate` which exported # the venv bin onto PATH; Architecture B's main-wrapper.sh does the @@ -337,27 +421,37 @@ ENV PATH="/opt/hermes/bin:/opt/hermes/.venv/bin:/opt/data/.local/bin:${PATH}" RUN mkdir -p /opt/data VOLUME [ "/opt/data" ] -# s6-overlay's /init is PID 1. It sets up the supervision tree, runs -# /etc/cont-init.d/* (our stage2 hook), starts s6-rc services -# declared in /etc/s6-overlay/s6-rc.d/, then exec's its remaining -# argv as the container's "main program" with stdin/stdout/stderr -# inherited (this is what makes interactive --tui work). When the -# main program exits, /init begins stage 3 shutdown and the container -# exits with the program's exit code. Replaces tini — see Phase 2 of -# docs/plans/2026-05-07-s6-overlay-dynamic-subagent-gateways.md. +# The image ENTRYPOINT is a tiny dispatcher rather than `/init` directly. +# When the image really owns PID 1 (normal Docker / Podman), the dispatcher +# execs `/init` and preserves the full s6 supervision tree. When a platform +# wraps the image entrypoint under its own PID-1 init (Fly Machines, +# `docker run --init`, some schedulers), `/init` would abort with +# `can only run as pid 1`; in that case the dispatcher falls back to +# `stage2-hook.sh` + `main-wrapper.sh` directly so foreground commands still +# work. See #38349. +# +# On the PID-1 path, s6-overlay's /init sets up the supervision tree, runs +# /etc/cont-init.d/* (our stage2 hook), starts s6-rc services declared in +# /etc/s6-overlay/s6-rc.d/, then exec's its remaining argv as the container's +# "main program" with stdin/stdout/stderr inherited (this is what makes +# interactive --tui work). When the main program exits, /init begins stage 3 +# shutdown and the container exits with the program's exit code. Replaces +# tini — see Phase 2 of docs/plans/2026-05-07-s6-overlay-dynamic-subagent-gateways.md. # # We use the ENTRYPOINT+CMD split rather than CMD alone so the # wrapper is prepended to user-supplied args automatically: # -# docker run → /init main-wrapper.sh (CMD default) -# docker run chat -q "hi" → /init main-wrapper.sh chat -q hi -# docker run sleep infinity → /init main-wrapper.sh sleep infinity -# docker run --tui → /init main-wrapper.sh --tui +# docker run → entrypoint-dispatch.sh (CMD default) +# docker run chat -q "hi" → entrypoint-dispatch.sh chat -q hi +# docker run sleep infinity → entrypoint-dispatch.sh sleep infinity +# docker run --tui → entrypoint-dispatch.sh --tui # # main-wrapper.sh handles arg routing (bare-exec vs. hermes # subcommand vs. no-args), drops to the hermes user via s6-setuidgid, # and exec's the final program so its exit code becomes the container -# exit code. Without the wrapper-as-ENTRYPOINT, leading-dash args -# like `--version` would be intercepted by /init's POSIX shell. -ENTRYPOINT [ "/init", "/opt/hermes/docker/main-wrapper.sh" ] +# exit code. The dispatcher preserves that contract across both the +# supervised PID-1 path and the non-PID-1 fallback path. Without the +# wrapper-as-ENTRYPOINT, leading-dash args like `--version` would be +# intercepted by /init's POSIX shell. +ENTRYPOINT [ "/opt/hermes/docker/entrypoint-dispatch.sh" ] CMD [ ] diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ffd55a6a82b1..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -graft skills -graft optional-skills -graft optional-mcps -graft locales -graft plugins -recursive-include gateway/assets * -global-exclude __pycache__ -global-exclude *.py[cod] diff --git a/README.md b/README.md index d78d6d57f045..c05112266746 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Use any model you want — [Nous Portal](https://portal.nousresearch.com), OpenR A closed learning loopAgent-curated memory with periodic nudges. Autonomous skill creation after complex tasks. Skills self-improve during use. FTS5 session search with LLM summarization for cross-session recall. Honcho dialectic user modeling. Compatible with the agentskills.io open standard. Scheduled automationsBuilt-in cron scheduler with delivery to any platform. Daily reports, nightly backups, weekly audits — all in natural language, running unattended. Delegates and parallelizesSpawn isolated subagents for parallel workstreams. Write Python scripts that call tools via RPC, collapsing multi-step pipelines into zero-context-cost turns. -Runs anywhere, not just your laptopSix terminal backends — local, Docker, SSH, Singularity, Modal, and Daytona. Daytona and Modal offer serverless persistence — your agent's environment hibernates when idle and wakes on demand, costing nearly nothing between sessions. Run it on a $5 VPS or a GPU cluster. +Runs anywhere, not just your laptopSeven terminal backends — local, Docker, SSH, Singularity, Modal, Daytona, and Vercel Sandbox. Daytona and Modal offer serverless persistence — your agent's environment hibernates when idle and wakes on demand, costing nearly nothing between sessions. Run it on a $5 VPS or a GPU cluster. Research-readyBatch trajectory generation, trajectory compression for training the next generation of tool-calling models. diff --git a/SECURITY.es.md b/SECURITY.es.md index 30b43716ebbb..086656d4f73f 100644 --- a/SECURITY.es.md +++ b/SECURITY.es.md @@ -173,9 +173,13 @@ modelo de autorización, pero las reglas a continuación se aplican uniformement **Superficies en Hermes Agent:** -- **Adaptadores de plataforma del gateway.** Integraciones de mensajería en - `gateway/platforms/` (Telegram, Discord, Slack, email, SMS, etc.) - y adaptadores análogos incluidos como plugins. +- **Adaptadores de plataforma del gateway.** La mayoría de las integraciones + de mensajería se distribuyen como plugins empaquetados en + `plugins/platforms//` (Telegram, Discord, Slack, email, SMS, etc.). + Los tipos base compartidos y un conjunto menor de adaptadores + legacy/directos viven en `gateway/platforms/` (`base.py`, Signal, servidor + API, webhooks, …), con descubrimiento y carga diferida vía + `gateway/platform_registry.py`. - **Superficies HTTP expuestas en red.** El adaptador del servidor API, el plugin del dashboard, los endpoints HTTP del plugin kanban, y cualquier otro plugin que vincule un socket de escucha. diff --git a/SECURITY.md b/SECURITY.md index 2579c6eaec56..7f5fd42db7b4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -177,9 +177,12 @@ authorization model, but the rules below apply uniformly. **Surfaces in Hermes Agent:** -- **Gateway platform adapters.** Messaging integrations in - `gateway/platforms/` (Telegram, Discord, Slack, email, SMS, etc.) - and analogous adapters shipped as plugins. +- **Gateway platform adapters.** Most messaging integrations ship as + bundled plugins under `plugins/platforms//` (Telegram, Discord, + Slack, email, SMS, etc.). Shared base types and a smaller set of + legacy/direct adapters live under `gateway/platforms/` + (`base.py`, Signal, API server, webhooks, …), with discovery and + deferred loading via `gateway/platform_registry.py`. - **Network-exposed HTTP surfaces.** The API server adapter, the dashboard plugin, the kanban plugin's HTTP endpoints, and any other plugin that binds a listening socket. diff --git a/acp_adapter/entry.py b/acp_adapter/entry.py index 55773536122a..7b549006bc27 100644 --- a/acp_adapter/entry.py +++ b/acp_adapter/entry.py @@ -32,6 +32,7 @@ import argparse import asyncio import logging +import os import sys from pathlib import Path from hermes_constants import get_hermes_home @@ -246,16 +247,24 @@ def main(argv: list[str] | None = None) -> None: import acp from .server import HermesACPAgent - # MCP tool discovery from config.yaml — run before asyncio.run() so - # it's safe to use blocking waits. (ACP also registers per-session - # MCP servers dynamically via asyncio.to_thread inside the event - # loop; that path is unaffected.) Moved from model_tools.py module - # scope to avoid freezing the gateway's loop on lazy import (#16856). - try: - from tools.mcp_tool import discover_mcp_tools - discover_mcp_tools() - except Exception: - logger.debug("MCP tool discovery failed at ACP startup", exc_info=True) + # MCP tool discovery from config.yaml — fire-and-forget in a + # background daemon thread so the ACP server becomes responsive + # immediately while MCP servers connect. Previously this blocked + # asyncio.run() for 2-5 s. (ACP also registers per-session MCP + # servers dynamically via asyncio.to_thread inside the event loop; + # that path is unaffected.) Moved from model_tools.py module scope + # to avoid freezing the gateway's loop on lazy import (#16856). + # Metadata-only hosts can opt out of unrelated global MCP startup. + if os.environ.get("HERMES_ACP_SKIP_CONFIGURED_MCP", "").strip() != "1": + try: + from hermes_cli.mcp_startup import start_background_mcp_discovery + + start_background_mcp_discovery( + logger=logger, + thread_name="acp-mcp-discovery", + ) + except Exception: + logger.debug("MCP tool discovery failed at ACP startup", exc_info=True) agent = HermesACPAgent() try: diff --git a/acp_adapter/permissions.py b/acp_adapter/permissions.py index 5f29a96725cc..b10b2a169ec5 100644 --- a/acp_adapter/permissions.py +++ b/acp_adapter/permissions.py @@ -158,9 +158,16 @@ def _callback( try: response = future.result(timeout=timeout) - except (FutureTimeout, Exception) as exc: + except FutureTimeout: future.cancel() - logger.warning("Permission request timed out or failed: %s", exc) + logger.warning("Permission request timed out after %ss", timeout) + # Distinct from an explicit deny: the client never answered. + # tools.approval callers report this as "timed out without user + # response" instead of a user denial. + return "timeout" + except Exception as exc: + future.cancel() + logger.warning("Permission request failed: %s", exc) return "deny" if response is None: diff --git a/acp_adapter/server.py b/acp_adapter/server.py index 3e79bdcd38a4..44d1df5ecc98 100644 --- a/acp_adapter/server.py +++ b/acp_adapter/server.py @@ -78,6 +78,7 @@ COMPRESSED_SUMMARY_METADATA_KEY, ContextCompressor, ) +from agent.interrupt_compat import request_hard_interrupt from tools.approval import ( reset_hermes_interactive_context, set_hermes_interactive_context, @@ -85,6 +86,110 @@ logger = logging.getLogger(__name__) + +def _named_custom_provider_catalogs() -> list[tuple[str, str, list[tuple[str, str]]]]: + """Return ``(slug, label, [(model_id, description), ...])`` for named endpoints. + + Covers both the v12 ``providers:`` mapping and the legacy + ``custom_providers:`` list. These endpoints never appear in canonical + provider enumeration, so without this the ACP model selector hides every + named endpoint that the TUI ``/model`` picker already renders (#47039 + implemented named-endpoint rows for the TUI surface only). + + Model lists come from the entry's declared models (``default_model`` + + ``models``), refreshed from the endpoint's live ``/models`` listing when a + credential is available and ``discover_models`` is not disabled. Declared + models are kept even when live discovery fails — some OpenAI-compatible + endpoints (e.g. Bedrock Mantle Responses) expose no ``/models`` route at + all yet serve the declared models fine. + + Slugs use the ``custom:`` shape that ``parse_model_input`` and + ``resolve_runtime_provider`` already resolve, so encoded choice ids + (``custom::``) round-trip through ``set_session_model`` + unchanged. + """ + try: + from hermes_cli.config import ( + get_compatible_custom_providers, + is_provider_enabled, + load_config, + ) + from hermes_cli.models import fetch_api_models + from hermes_cli.providers import custom_provider_slug + except ImportError: + return [] + + try: + cfg = load_config() + entries = get_compatible_custom_providers(cfg) + except Exception: + logger.debug("Could not load named custom providers", exc_info=True) + return [] + + # ``get_compatible_custom_providers`` drops the ``enabled`` flag during + # normalization, so collect explicitly disabled provider keys from the + # raw config and skip their entries below. + disabled_keys: set[str] = set() + raw_providers = cfg.get("providers") if isinstance(cfg, dict) else None + if isinstance(raw_providers, dict): + for raw_key, raw_entry in raw_providers.items(): + if isinstance(raw_entry, dict) and not is_provider_enabled(raw_entry): + disabled_keys.add(str(raw_key).strip().lower()) + + catalogs: list[tuple[str, str, list[tuple[str, str]]]] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + provider_key = str(entry.get("provider_key", "") or "").strip() + if provider_key.lower() in disabled_keys: + continue + name = str(entry.get("name", "") or "").strip() + base_url = str(entry.get("base_url", "") or "").strip() + if not name or not base_url: + continue + slug = custom_provider_slug(name, provider_key) + + api_key = str(entry.get("api_key", "") or "").strip() + if not api_key: + key_env = str(entry.get("key_env", "") or "").strip() + api_key = os.environ.get(key_env, "").strip() if key_env else "" + + declared: list[str] = [] + default_model = str(entry.get("model", "") or "").strip() + if default_model: + declared.append(default_model) + models_cfg = entry.get("models") + if isinstance(models_cfg, dict): + for mid in models_cfg: + mid = str(mid or "").strip() + if mid and mid not in declared: + declared.append(mid) + + if not api_key and not declared: + # No credential to discover with and nothing declared: + # not addressable from the selector. + continue + + model_ids = list(declared) + discover = entry.get("discover_models", True) + if isinstance(discover, str): + discover = discover.lower() not in {"false", "no", "0"} + if discover and api_key: + try: + live = fetch_api_models( + api_key, base_url, api_mode=entry.get("api_mode") + ) + except Exception: + live = None + if live: + model_ids = declared + [m for m in live if m not in declared] + + if not model_ids: + continue + catalogs.append((slug, name, [(mid, "") for mid in model_ids])) + + return catalogs + try: from hermes_cli import __version__ as HERMES_VERSION except Exception: @@ -97,6 +202,13 @@ # does not expose a client-side limit, so this is a fixed cap that clients # paginate against using `cursor` / `next_cursor`. _LIST_SESSIONS_PAGE_SIZE = 50 +# Per-provider cap for the ACP model selector. ACP clients (Zed, Buzz) render +# the whole `availableModels` array in one dropdown, so an unbounded +# cross-provider catalog degrades the picker. Mirrors the cap the MoA picker +# already uses (`hermes_cli/moa_cmd.py`). This bounds each provider's row, not +# the total; aggregator providers stay intentionally uncapped inside the shared +# inventory, and the current model is always kept via the fallback insert below. +ACP_MAX_MODELS_PER_PROVIDER = 200 _MAX_ACP_RESOURCE_BYTES = 512 * 1024 _TEXT_RESOURCE_MIME_PREFIXES = ("text/",) _TEXT_RESOURCE_MIME_TYPES = { @@ -585,46 +697,108 @@ def _encode_model_choice(provider: str | None, model: str | None) -> str: return f"{raw_provider}:{raw_model}" def _build_model_state(self, state: SessionState) -> SessionModelState | None: - """Return the ACP model selector payload for editors like Zed.""" + """Return authenticated providers and their models for ACP clients. + + The shared Hermes inventory is also used by ``hermes model``, the TUI, + and the dashboard. Keeping ACP on that substrate prevents its selector + from silently collapsing to the current provider's curated list. + """ model = str(state.model or getattr(state.agent, "model", "") or "").strip() provider = getattr(state.agent, "provider", None) or detect_provider() or "openrouter" try: - from hermes_cli.models import curated_models_for_provider, normalize_provider, provider_label + from hermes_cli.inventory import build_models_payload, load_picker_context + from hermes_cli.models import normalize_provider, provider_label normalized_provider = normalize_provider(provider) - provider_name = provider_label(normalized_provider) + context = load_picker_context().with_overrides( + current_provider=normalized_provider, + current_model=model, + current_base_url=str(getattr(state.agent, "base_url", "") or ""), + ) + payload = build_models_payload( + context, + explicit_only=True, + include_unconfigured=False, + picker_hints=False, + canonical_order=True, + pricing=False, + capabilities=False, + refresh=False, + probe_custom_providers=False, + probe_current_custom_provider=False, + max_models=ACP_MAX_MODELS_PER_PROVIDER, + ) + available_models: list[ModelInfo] = [] seen_ids: set[str] = set() - - for model_id, description in curated_models_for_provider(normalized_provider): - rendered_model = str(model_id or "").strip() - if not rendered_model: - continue - choice_id = self._encode_model_choice(normalized_provider, rendered_model) - if choice_id in seen_ids: + for row in payload.get("providers") or []: + row_provider = normalize_provider(str(row.get("slug") or "").strip()) + if not row_provider: continue - desc_parts = [f"Provider: {provider_name}"] - if description: - desc_parts.append(str(description).strip()) - if rendered_model == model: - desc_parts.append("current") - available_models.append( - ModelInfo( - model_id=choice_id, - name=rendered_model, - description=" • ".join(part for part in desc_parts if part), - ) + provider_name = str(row.get("name") or "").strip() or provider_label( + row_provider ) - seen_ids.add(choice_id) + for model_entry in row.get("models") or []: + if isinstance(model_entry, dict): + rendered_model = str( + model_entry.get("id") + or model_entry.get("model") + or model_entry.get("name") + or "" + ).strip() + else: + rendered_model = str(model_entry or "").strip() + if not rendered_model: + continue + choice_id = self._encode_model_choice(row_provider, rendered_model) + if choice_id in seen_ids: + continue + is_current = ( + row_provider == normalized_provider and rendered_model == model + ) + description = f"Provider: {provider_name}" + if is_current: + description += " • current" + available_models.append( + ModelInfo( + model_id=choice_id, + name=f"{provider_name} · {rendered_model}", + description=description, + ) + ) + seen_ids.add(choice_id) + + # Named user-defined endpoints (providers: / custom_providers:) + # are invisible to canonical provider enumeration — append them + # so editor clients can select them like the TUI /model picker. + for named_slug, named_label, named_catalog in _named_custom_provider_catalogs(): + for named_model, named_desc in named_catalog: + named_choice = self._encode_model_choice(named_slug, named_model) + if not named_choice or named_choice in seen_ids: + continue + named_parts = [f"Provider: {named_label}"] + if named_desc: + named_parts.append(str(named_desc).strip()) + if named_slug == normalized_provider and named_model == model: + named_parts.append("current") + available_models.append( + ModelInfo( + model_id=named_choice, + name=named_model, + description=" • ".join(part for part in named_parts if part), + ) + ) + seen_ids.add(named_choice) current_model_id = self._encode_model_choice(normalized_provider, model) if current_model_id and current_model_id not in seen_ids: + provider_name = provider_label(normalized_provider) available_models.insert( 0, ModelInfo( model_id=current_model_id, - name=model, + name=f"{provider_name} · {model}", description=f"Provider: {provider_name} • current", ), ) @@ -864,6 +1038,102 @@ async def _register_session_mcp_servers( exc_info=True, ) + def _schedule_mcp_late_refresh(self, state: SessionState) -> None: + """Refresh the agent's tool snapshot when background MCP discovery lands late. + + ACP entry.py starts MCP tool discovery in a background daemon thread so a + slow/dead configured server can't block ``asyncio.run()``. ``_make_agent`` + briefly joins that thread (``wait_for_mcp_discovery``, bounded ~1.5s) so + already-spawning fast servers land in the snapshot — but a server slower + than the bound lands *after* the agent is built, leaving its tools absent + for the whole session. + + This schedules an off-critical-path daemon that waits for discovery to + finish (bounded 30s), then rebuilds the snapshot via the shared + ``refresh_agent_mcp_tools`` helper — the same rebuild ``/reload-mcp`` + performs, but automatic. Mirrors the TUI late-refresh (PR #48403). + + Cache safety: the rebuild only runs while the session is still + pre-first-turn (no API call made yet → nothing cached to invalidate). + Once the user has sent a message we leave the snapshot frozen rather + than break the cached prompt prefix mid-conversation; servers that land + later are picked up cache-safely by the between-turns prologue refresh + (``agent/turn_context.py``) at the next turn boundary. The marginal + value of this pre-first-turn daemon is therefore freshness in the + window [session created → first message] — e.g. the "Available tools" + listing a client may request before the first prompt. + No-op when discovery already finished, when the join times out, when the + registry was unchanged, or when the session was closed while waiting. + """ + try: + from hermes_cli.mcp_startup import mcp_discovery_in_flight + except Exception: + return + if not mcp_discovery_in_flight(): + return + + import threading + + agent = state.agent + session_id = state.session_id + + def _wait_then_refresh() -> None: + try: + from hermes_cli.mcp_startup import join_mcp_discovery + + if not join_mcp_discovery(timeout=30.0): + return + + # Session may have been closed while we waited. In-memory-only + # lookup on purpose: ``get_session()`` falls through to a DB + # restore that builds a whole new AIAgent as a side effect just + # to decide "no-op" here (the TUI equivalent also checks its + # in-memory dict only). + with self.session_manager._lock: + current = self.session_manager._sessions.get(session_id) + if current is None or current.agent is not agent: + return + + # Cache safety: never rebuild the tool list once the conversation + # has started — that would invalidate the cached prompt prefix. + # Serialized with turn start: ``prompt()`` flips ``is_running`` + # under ``runtime_lock`` before dispatching, so holding it here + # (and bailing when a turn is already running) closes the window + # where the guard passes but the first prompt starts before the + # refresh publishes — which would swap ``tools=`` mid-turn and + # break the just-created cache prefix. + with current.runtime_lock: + if current.is_running: + return + if ( + int(getattr(agent, "_user_turn_count", 0) or 0) > 0 + or int(getattr(agent, "_api_call_count", 0) or 0) > 0 + ): + return + + from tools.mcp_tool import refresh_agent_mcp_tools + + added = refresh_agent_mcp_tools(agent, quiet_mode=True) + if added: + logger.info( + "Session %s: late MCP refresh added %d tools: %s", + session_id, + len(added), + ", ".join(sorted(added)), + ) + except Exception: + logger.debug( + "Session %s: late MCP refresh failed", + session_id, + exc_info=True, + ) + + threading.Thread( + target=_wait_then_refresh, + name=f"acp-mcp-late-refresh-{session_id}", + daemon=True, + ).start() + # ---- ACP lifecycle ------------------------------------------------------ async def initialize( @@ -1170,6 +1440,7 @@ async def new_session( ) -> NewSessionResponse: state = self.session_manager.create_session(cwd=cwd) await self._register_session_mcp_servers(state, mcp_servers) + self._schedule_mcp_late_refresh(state) logger.info("New session %s (cwd=%s)", state.session_id, cwd) self._schedule_available_commands_update(state.session_id) self._schedule_usage_update(state) @@ -1194,6 +1465,7 @@ async def load_session( logger.warning("load_session: session %s not found", session_id) return None await self._register_session_mcp_servers(state, mcp_servers) + self._schedule_mcp_late_refresh(state) logger.info("Loaded session %s", session_id) # Per ACP spec, `session/load` must stream the prior conversation back # to the client via `session/update` notifications BEFORE responding, @@ -1241,6 +1513,7 @@ async def resume_session( logger.warning("resume_session: session %s not found, creating new", session_id) state = self.session_manager.create_session(cwd=cwd) await self._register_session_mcp_servers(state, mcp_servers) + self._schedule_mcp_late_refresh(state) logger.info("Resumed session %s", state.session_id) # See `load_session` above for the spec rationale — replay must # complete before the response so clients receive the full transcript @@ -1275,8 +1548,8 @@ async def cancel(self, session_id: str, **kwargs: Any) -> None: # redirectable work. state.cancel_event.set() try: - if getattr(state, "agent", None) and hasattr(state.agent, "interrupt"): - state.agent.interrupt() + if getattr(state, "agent", None): + request_hard_interrupt(state.agent) except Exception: logger.debug( "Failed to interrupt ACP session %s", @@ -1588,7 +1861,19 @@ def _run_agent() -> dict: clear_session_vars, set_session_vars, ) - session_tokens = set_session_vars(session_key=session_id) + # ``cwd`` pins the logical working directory for this context, + # which is what the system prompt's "Current working directory" + # line reports (agent/prompt_builder.py -> resolve_agent_cwd). + # Without it the prompt advertises the global Hermes workspace + # while the tools are rooted at the client's project, so the + # model emits absolute paths under ~/.hermes/workspace and the + # edit silently lands outside the editor's workspace. + # cron_session="" explicitly marks this as a non-cron context, + # masking any leaked process-global HERMES_CRON_SESSION (#37968). + session_tokens = set_session_vars( + session_key=session_id, session_id=session_id, cwd=state.cwd, + cron_session="", + ) except Exception: session_tokens = None clear_session_vars = None # type: ignore[assignment] @@ -1875,8 +2160,26 @@ def _handle_slash_command(self, text: str, state: SessionState) -> str | None: if handler is None: return None # not a known command — let the LLM handle it - try: + # Slash handlers run on the event-loop thread, OUTSIDE the per-turn + # contextvars.copy_context() that pins the session cwd for the agent + # call. ``/compress`` and ``/model`` reach code that REBUILDS the + # system prompt (agent._build_system_prompt -> resolve_agent_cwd), so + # an unpinned handler bakes the Hermes install tree into the session's + # cached prompt — persisted, and therefore poisoning every later turn + # even though the turn itself is pinned. Pin inside a fresh context so + # the write can't leak into other concurrent ACP sessions and needs no + # teardown. + def _dispatch() -> str | None: + try: + from agent.runtime_cwd import set_session_cwd + + set_session_cwd(state.cwd) + except Exception: + logger.debug("Could not pin ACP session cwd for slash command", exc_info=True) return handler(args, state) + + try: + return contextvars.copy_context().run(_dispatch) except Exception as e: logger.error("Slash command /%s error: %s", cmd, e, exc_info=True) return f"Error executing /{cmd}: {e}" diff --git a/acp_adapter/session.py b/acp_adapter/session.py index 6f1e17a07f57..6e16016a6b3b 100644 --- a/acp_adapter/session.py +++ b/acp_adapter/session.py @@ -648,6 +648,30 @@ def _make_agent( logger.debug("ACP session falling back to default provider resolution", exc_info=True) _register_task_cwd(session_id, cwd) + + # Bounded wait for background MCP discovery so already-spawning fast + # servers land in the agent's tool snapshot. ACP entry.py fires + # discovery in a background daemon thread (start_background_mcp_discovery); + # the agent snapshots tools once at build (run_agent/agent_init) and + # never re-reads the registry, so without this join a reachable-but- + # slow configured server would be invisible for the whole session. + # ``ensure_mcp_discovery_before_agent_build`` also (re)starts discovery + # when the entry.py spawn never ran or exited with zero connected + # servers (the retry-after-zero-connected allowance), making this + # construction site self-sufficient. Bounded by + # ``mcp_discovery_timeout`` (config.yaml, default ~1.5s) so a dead + # server can't block — servers that miss the bound are picked up by + # the automatic late-refresh (see HermesACPAgent._schedule_mcp_late_refresh). + try: + from hermes_cli.mcp_startup import ensure_mcp_discovery_before_agent_build + + ensure_mcp_discovery_before_agent_build( + logger=logger, + thread_name="acp-mcp-discovery", + ) + except Exception: + logger.debug("ACP: bounded MCP discovery wait failed", exc_info=True) + agent = AIAgent(**kwargs) # Codex app-server sessions are spawned lazily on the first turn. Stamp # the ACP workspace onto the agent so the Codex runtime starts from the diff --git a/agent/agent_init.py b/agent/agent_init.py index bfa2808de0a3..ad6b0eb00af5 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -33,6 +33,7 @@ from agent.context_compressor import ContextCompressor from agent.iteration_budget import IterationBudget from agent.memory_manager import StreamingContextScrubber +from agent.session_activity import ActivityProvenance from agent.model_metadata import ( MINIMUM_CONTEXT_LENGTH, fetch_model_metadata, @@ -118,7 +119,7 @@ def _provider_default_routes(provider: str) -> set[str]: from hermes_cli.providers import HERMES_OVERLAYS, get_provider overlay = HERMES_OVERLAYS.get(provider) - provider_def = get_provider(provider) + provider_def = get_provider(provider, allow_network=False) for value in ( getattr(overlay, "base_url_override", ""), getattr(provider_def, "base_url", ""), @@ -456,7 +457,6 @@ def init_agent( args: list[str] | None = None, model: str = "", max_iterations: int = 90, # Default tool-calling iterations (shared with subagents) - tool_delay: float = 1.0, enabled_toolsets: List[str] = None, disabled_toolsets: List[str] = None, save_trajectories: bool = False, @@ -530,7 +530,6 @@ def init_agent( api_mode (str): API mode override: "chat_completions" or "codex_responses" model (str): Model name to use (default: "anthropic/claude-opus-4.6") max_iterations (int): Maximum number of tool calling iterations (default: 90) - tool_delay (float): Delay between tool calls in seconds (default: 1.0) enabled_toolsets (List[str]): Only enable tools from these toolsets (optional) disabled_toolsets (List[str]): Disable tools from these toolsets (optional) save_trajectories (bool): Whether to save conversation trajectories to JSONL files (default: False) @@ -576,7 +575,6 @@ def init_agent( # Shared iteration budget — parent creates, children inherit. # Consumed by every LLM turn across parent + all subagents. agent.iteration_budget = iteration_budget or IterationBudget(max_iterations) - agent.tool_delay = tool_delay agent.save_trajectories = save_trajectories agent.verbose_logging = verbose_logging agent.quiet_mode = quiet_mode @@ -645,6 +643,13 @@ def init_agent( # AWS Bedrock — auto-detect from provider name or base URL # (bedrock-runtime..amazonaws.com). agent.api_mode = "bedrock_converse" + elif agent.provider in {"nous", "nous-portal", "nousresearch"}: + # Portal is dual-wire: anthropic/* → Messages, everything else → + # chat_completions. Callers that already pass api_mode win above; + # this covers direct AIAgent construction without a resolved runtime. + from hermes_cli.providers import nous_api_mode + + agent.api_mode = nous_api_mode(agent.model) else: agent.api_mode = "chat_completions" @@ -761,6 +766,9 @@ def init_agent( # Interrupt mechanism for breaking out of tool loops agent._interrupt_requested = False agent._interrupt_message = None # Optional message that triggered interrupt + # Explicit hard cancellation is separate from redirect/message state. A + # thread-safe Event makes the cause atomic for auxiliary stream pollers. + agent._hard_interrupt_requested = threading.Event() agent._execution_thread_id: int | None = None # Set at run_conversation() start agent._interrupt_thread_signal_pending = False agent._client_lock = threading.RLock() @@ -823,24 +831,41 @@ def init_agent( # Anthropic prompt caching: auto-enabled for Claude models on native # Anthropic, OpenRouter, and third-party gateways that speak the # Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces - # input costs by ~75% on multi-turn conversations. Uses system_and_3 - # strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy`` - # for the layout-vs-transport decision. + # input costs by ~75% on multi-turn conversations. Uses four breakpoints: + # the static system prefix, full system prompt, and last two messages + # (falling back to system-and-3 when no static prefix is available). See + # ``_anthropic_prompt_cache_policy`` for the layout-vs-transport decision. agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy() ) + agent._cache_disabled = False # Anthropic supports "5m" (default) and "1h" cache TTL tiers. Read from # config.yaml under prompt_caching.cache_ttl; unknown values keep "5m". # 1h tier costs 2x on write vs 1.25x for 5m, but amortizes across long # sessions with >5-minute pauses between turns (#14971). + # + # Setting cache_ttl to a falsy value (false / null / "off" / "disabled" / + # "no" / "none") disables prompt caching entirely. This is useful for + # OAuth subscription users where cache writes bill against "extra usage" + # or for third-party proxies that inject their own cache_control markers + # (#13477). The disable propagates through anthropic_prompt_cache_policy() + # and restore_primary_runtime() so it survives /model switches and + # fallback re-derivation (#33555). agent._cache_ttl = "5m" try: - from hermes_cli.config import load_config as _load_pc_cfg + from hermes_cli.config import load_config_readonly as _load_pc_cfg + + from agent.agent_runtime_helpers import cache_ttl_means_disabled _pc_cfg = _load_pc_cfg().get("prompt_caching", {}) or {} _ttl = _pc_cfg.get("cache_ttl", "5m") if _ttl in {"5m", "1h"}: agent._cache_ttl = _ttl + elif cache_ttl_means_disabled(_ttl): + agent._use_prompt_caching = False + agent._use_native_cache_layout = False + agent._cache_ttl = None + agent._cache_disabled = True except Exception: pass @@ -859,6 +884,11 @@ def init_agent( # notifications to show progress. agent._last_activity_ts: float = time.time() agent._last_activity_desc: str = "initializing" + # Default / unmigrated paths and _touch_activity stamp unknown; named + # provenances are stamped by compression writers (heartbeat / timeout / cooldown). + agent._last_activity_provenance = ActivityProvenance.UNKNOWN + # Rate-limit durable SessionDB activity stamps from _touch_activity (#72016). + agent._session_activity_last_persist_mono: float = 0.0 agent._current_tool: str | None = None agent._api_call_count: int = 0 # Opt-out flag for the between-turns MCP tool refresh (build_turn_context). @@ -879,8 +909,10 @@ def init_agent( # report cumulative micros spent. Surfaced behind HERMES_DEV_CREDITS. agent._credits_state = None agent._credits_session_start_micros = None - # Threshold-notice latch (L4): active sticky-notice keys + the warn90 crossing gate. - agent._credits_latch = {"active": set(), "seen_below_90": False, "usage_band": None} + # Threshold-notice latch (L4): active sticky-notice keys + the crossing gates. + from agent.credits_tracker import new_credits_latch + + agent._credits_latch = new_credits_latch() # OpenRouter response cache hit counter — incremented when # X-OpenRouter-Cache-Status: HIT is seen in streaming response headers. @@ -950,12 +982,6 @@ def init_agent( agent._stream_writer_tls = threading.local() agent._stream_writer_dropped = 0 - # Displayed reasoning text streamed during the current model response, - # captured only when a surface consumed it via a reasoning callback. Used - # by active-turn redirect to checkpoint what the user actually saw without - # ever persisting hidden provider reasoning. - agent._current_streamed_reasoning_text = "" - # Optional current-turn user-message override used when the API-facing # user message intentionally differs from the persisted transcript # (e.g. CLI voice mode adds a temporary prefix for the live call only). @@ -1089,7 +1115,7 @@ def init_agent( # Guardrail config — read from config.yaml at init time. agent._bedrock_guardrail_config = None try: - from hermes_cli.config import load_config as _load_br_cfg + from hermes_cli.config import load_config_readonly as _load_br_cfg _gr = _load_br_cfg().get("bedrock", {}).get("guardrail", {}) if _gr.get("guardrail_identifier") and _gr.get("guardrail_version"): agent._bedrock_guardrail_config = { @@ -1154,10 +1180,14 @@ def init_agent( elif base_url_host_matches(effective_base, "chatgpt.com"): from agent.auxiliary_client import _codex_cloudflare_headers client_kwargs["default_headers"] = _codex_cloudflare_headers(api_key) + elif base_url_host_matches(effective_base, "x.ai"): + from tools.xai_http import hermes_xai_default_headers + + client_kwargs["default_headers"] = hermes_xai_default_headers() elif "default_headers" not in client_kwargs: # Fall back to profile.default_headers for providers that - # declare custom headers (e.g. Kimi User-Agent on non-kimi.com - # endpoints). + # declare custom headers (e.g. Vercel AI Gateway attribution, + # Kimi User-Agent on non-kimi.com endpoints). try: from providers import get_provider_profile as _gpf _ph = _gpf(agent.provider) @@ -1216,16 +1246,20 @@ def init_agent( _fb_entries = [fallback_model] _fb_resolved = False for _fb in _fb_entries: - _fb_explicit_key = (_fb.get("api_key") or "").strip() or None - if not _fb_explicit_key: - _fb_key_env = (_fb.get("key_env") or _fb.get("api_key_env") or "").strip() - if _fb_key_env: - _fb_explicit_key = os.getenv(_fb_key_env, "").strip() or None - _fb_client, _fb_model = resolve_provider_client( - _fb["provider"], model=_fb["model"], raw_codex=True, - explicit_base_url=_fb.get("base_url"), - explicit_api_key=_fb_explicit_key, - ) + try: + from hermes_cli.fallback_config import resolve_entry_api_key + _fb_explicit_key = resolve_entry_api_key(_fb) + _fb_client, _fb_model = resolve_provider_client( + _fb["provider"], model=_fb["model"], raw_codex=True, + explicit_base_url=_fb.get("base_url"), + explicit_api_key=_fb_explicit_key, + ) + except Exception as _fb_exc: + logger.debug( + "Init-time fallback entry %s failed: %s", + _fb.get("provider"), _fb_exc, + ) + continue if _fb_client is not None: agent.provider = _fb["provider"] agent.model = _fb_model or _fb["model"] @@ -1341,6 +1375,13 @@ def init_agent( print("⚠️ Warning: API key appears invalid or missing") except Exception as e: raise RuntimeError(f"Failed to initialize OpenAI client: {e}") + + # Keep a stable identity for the pool entry that supplied this runtime. + # OAuth refreshes can replace the runtime token before a failed request is + # recovered, so the mutable API-key value alone cannot reliably attribute + # the failure to its source entry. + from agent.agent_runtime_helpers import sync_credential_pool_entry_id + sync_credential_pool_entry_id(agent) # Provider fallback chain — ordered list of backup providers tried # when the primary is exhausted (rate-limit, overload, connection @@ -1453,7 +1494,17 @@ def init_agent( set_current_session_id(agent.session_id) except Exception: - os.environ["HERMES_SESSION_ID"] = agent.session_id + # Preserve the root-agent legacy fallback, but never let delegated + # construction publish a child ID process-wide even if the ContextVar + # bridge itself failed to import. + try: + from agent.delegation_context import is_delegated_child_context + + delegated_child = is_delegated_child_context() + except Exception: + delegated_child = False + if not delegated_child: + os.environ["HERMES_SESSION_ID"] = agent.session_id # Session logs go into ~/.hermes/sessions/ alongside gateway sessions hermes_home = get_hermes_home() @@ -1465,7 +1516,7 @@ def init_agent( # reads the JSON files directly. See run_agent._save_session_log. agent._session_json_enabled = False try: - from hermes_cli.config import load_config as _load_sess_cfg + from hermes_cli.config import load_config_readonly as _load_sess_cfg _sess_cfg = (_load_sess_cfg().get("sessions") or {}) agent._session_json_enabled = bool(_sess_cfg.get("write_json_snapshots", False)) except Exception: @@ -1487,6 +1538,9 @@ def init_agent( # Cached system prompt -- built once per session, only rebuilt on compression agent._cached_system_prompt: Optional[str] = None + # Cross-session-stable prefix of the cached prompt. It remains separate + # from the persisted string and is used only to place an early cache marker. + agent._cached_system_prompt_static: Optional[str] = None # Filesystem checkpoint manager (transparent — not a tool) from tools.checkpoint_manager import CheckpointManager @@ -1526,6 +1580,17 @@ def init_agent( "reasoning_config": reasoning_config, "max_tokens": max_tokens, } + # Persist a process-scoped --yolo launch into the session row so a later + # `hermes --resume ` can restore the bypass (CLI resume paths read + # model_config.yolo_mode back via SessionDB.session_yolo_enabled). + # Session-scoped /yolo toggles persist separately through + # SessionDB.set_session_yolo at toggle time. + try: + from tools.approval import _YOLO_MODE_FROZEN + if _YOLO_MODE_FROZEN: + agent._session_init_model_config["yolo_mode"] = True + except Exception: + pass # In-memory todo list for task planning (one per agent/session) from tools.todo_tool import TodoStore @@ -1533,7 +1598,7 @@ def init_agent( # Load config once for memory, skills, and compression sections try: - from hermes_cli.config import load_config as _load_agent_config + from hermes_cli.config import load_config_readonly as _load_agent_config _agent_cfg = _load_agent_config() except Exception: _agent_cfg = {} @@ -1869,7 +1934,7 @@ def init_agent( compression_max_attempts = 3 compression_max_attempts = min(compression_max_attempts, 10) - def _parse_config_int(raw, default): + def _parse_prune_int(raw, default): # Same parser semantics as compression.max_attempts above: reject # booleans (bool subclasses int — YAML `true` would coerce to 1), # reject fractional floats rather than truncating them, accept @@ -1890,14 +1955,14 @@ def _parse_config_int(raw, default): # default, so an unset key is behavior-neutral). Negative values are # treated as disabled rather than erroring. compression_proactive_prune_tokens = max( - 0, _parse_config_int(_compression_cfg.get("proactive_prune_tokens", 0), 0) + 0, _parse_prune_int(_compression_cfg.get("proactive_prune_tokens", 0), 0) ) - compression_proactive_prune_min_chars = _parse_config_int( + compression_proactive_prune_min_chars = _parse_prune_int( _compression_cfg.get("proactive_prune_min_result_chars", 8000), 8000 ) compression_proactive_prune_min_reclaim = max( 0, - _parse_config_int( + _parse_prune_int( _compression_cfg.get("proactive_prune_min_reclaim_tokens", 4096), 4096 ), ) @@ -1941,8 +2006,37 @@ def _parse_config_int(raw, default): # parent_session_id chain, no `name #N` renumber). See #38763 and # agent/conversation_compression.py. Consumed by compress_context(), not the # compressor, so it rides on the agent. + # Default True must match DEFAULT_CONFIG["compression"]["in_place"] + # (#38763). default=False here previously flipped agents into rotation + # mode whenever the merged config omitted the key (partial configs, + # load_config failure → {}), re-arming the pre-lease drift abort. compression_in_place = is_truthy_value( - _compression_cfg.get("in_place"), default=False + _compression_cfg.get("in_place"), default=True + ) + # Opt-in (default False): a micro-compaction pass rewrites already-sent + # history every turn, which breaks the provider prompt-cache prefix on a + # per-turn cadence rather than at an episodic boundary. That is the cost + # `proactive_prune_min_reclaim_tokens` exists to amortize, so the feature + # stays off until an operator opts in and accepts the tradeoff. + compression_micro_compact = is_truthy_value( + _compression_cfg.get("micro_compact"), default=False + ) + # How often a pass runs, in completed turns. Each pass rewrites + # already-sent history and costs one prompt-cache break, so this is the + # dial for how often that cost is paid: 1 = every turn (most aggressive + # reclaim), 5 = one break per five turns. Clamped to >= 1. + compression_micro_compact_every_n_turns = max( + 1, + _parse_prune_int(_compression_cfg.get("micro_compact_every_n_turns", 1), 1), + ) + # Rolling-summary defrag threshold, in tokens. Lived on the compressor as + # a hardcoded attribute with no path from config until now. + compression_micro_compact_defrag_tokens = max( + 1, + _parse_prune_int( + _compression_cfg.get("micro_compact_defrag_threshold_tokens", 2000), + 2000, + ), ) codex_app_server_auto_compaction = str( _compression_cfg.get("codex_app_server_auto", "native") or "native" @@ -1958,10 +2052,7 @@ def _parse_config_int(raw, default): # this many seconds of inactivity (0 = disabled). Time-based, so it # complements the size-based threshold above. Consumed by build_turn_context(). compression_idle_compact_after_seconds = max( - 0, - _parse_config_int( - _compression_cfg.get("idle_compact_after_seconds", 0), 0 - ), + 0, int(_compression_cfg.get("idle_compact_after_seconds", 0)) ) # Read optional explicit context_length override for the auxiliary @@ -2259,7 +2350,18 @@ def _parse_config_int(raw, default): # AFTER the custom_providers branch so per-model overrides aren't lost. agent._config_context_length = _config_context_length - agent._ensure_lmstudio_runtime_loaded(_config_context_length) + _lmstudio_runtime_context_length = agent._ensure_lmstudio_runtime_loaded( + _config_context_length + ) + if agent._lmstudio_load_was_unverified(_lmstudio_runtime_context_length): + _ra().logger.warning( + "LM Studio model activation was rejected or completed without a " + "verifiable active context length; falling back to configured context" + ) + _effective_context_length = agent._effective_lmstudio_context_length( + _config_context_length, + _lmstudio_runtime_context_length, + ) @@ -2336,7 +2438,7 @@ def _parse_config_int(raw, default): agent.model, base_url=agent.base_url, api_key=getattr(agent, "api_key", ""), - config_context_length=_config_context_length, + config_context_length=_effective_context_length, provider=agent.provider, custom_providers=_custom_providers, ) @@ -2371,7 +2473,7 @@ def _parse_config_int(raw, default): quiet_mode=agent.quiet_mode, base_url=agent.base_url, api_key=getattr(agent, "api_key", ""), - config_context_length=_config_context_length, + config_context_length=_effective_context_length, provider=agent.provider, api_mode=agent.api_mode, abort_on_summary_failure=compression_abort_on_summary_failure, @@ -2391,9 +2493,18 @@ def _parse_config_int(raw, default): pass agent.compression_enabled = compression_enabled agent.compression_in_place = compression_in_place + # Apply micro-compaction settings to the compressor (feature is opt-in) + _cc = getattr(agent, "context_compressor", None) + if _cc is not None and hasattr(_cc, "_micro_compact_enabled"): + _cc._micro_compact_enabled = compression_micro_compact + if _cc is not None and hasattr(_cc, "_micro_compact_every_n_turns"): + _cc._micro_compact_every_n_turns = compression_micro_compact_every_n_turns + if _cc is not None and hasattr(_cc, "_micro_compact_defrag_threshold_tokens"): + _cc._micro_compact_defrag_threshold_tokens = ( + compression_micro_compact_defrag_tokens + ) agent.codex_app_server_auto_compaction = codex_app_server_auto_compaction agent.max_compression_attempts = compression_max_attempts - agent.compression_summary_target_ratio = compression_target_ratio agent.compression_idle_compact_after_seconds = ( compression_idle_compact_after_seconds ) @@ -2401,7 +2512,13 @@ def _parse_config_int(raw, default): # Reject models whose context window is below the minimum required # for reliable tool-calling workflows (64K tokens). _ctx = getattr(agent.context_compressor, "context_length", 0) - if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH: + _allow_lmstudio_explicit_below_floor = ( + str(getattr(agent, "provider", "") or "").strip().lower() == "lmstudio" + and isinstance(agent._config_context_length, int) + and not isinstance(agent._config_context_length, bool) + and agent._config_context_length > 0 + ) + if _ctx and _ctx < MINIMUM_CONTEXT_LENGTH and not _allow_lmstudio_explicit_below_floor: raise ValueError( f"Model {agent.model} has a context window of {_ctx:,} tokens, " f"which is below the minimum {MINIMUM_CONTEXT_LENGTH:,} required " diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index f9ff6b112efd..fab406a3eb3c 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -36,7 +36,7 @@ from agent.prompt_builder import format_steer_marker from agent.tool_dispatch_helpers import _trajectory_normalize_msg, make_tool_result_message from agent.trajectory import convert_scratchpad_to_think -from agent.credential_pool import STATUS_EXHAUSTED +from agent.credential_pool import STATUS_EXHAUSTED, credential_pool_matches_provider from agent.error_classifier import FailoverReason from agent.turn_context import drop_stale_api_content from utils import base_url_host_matches, base_url_hostname, env_var_enabled, atomic_json_write @@ -52,6 +52,45 @@ _MAX_AUTH_REFRESH_ATTEMPTS = 2 +_REASONING_TAG_NAMES = ("think", "thinking", "reasoning", "REASONING_SCRATCHPAD", "thought") +_TOOL_CALL_TAG_NAMES = ("tool_call", "tool_calls", "tool_result", "function_call", "function_calls") + +_REASONING_BLOCK_PATTERNS = tuple( + re.compile(rf"<{name}>.*?", re.DOTALL | re.IGNORECASE) + for name in _REASONING_TAG_NAMES +) + +_TOOL_CALL_BLOCK_PATTERNS = tuple( + re.compile(rf"<{name}\b[^>]*>.*?", re.DOTALL | re.IGNORECASE) + for name in _TOOL_CALL_TAG_NAMES +) + +# Named blocks — see strip_think_blocks step 1c for the +# full rationale (sentence-boundary lookbehind + tempered-dot body so a plain +# prose mention of "function" is never eaten). +_NAMED_FUNCTION_BLOCK_PATTERN = re.compile( + r'(?:(?<=^)|(?<=[\n\r.!?:]))[ \t]*' + r']*\bname\s*=[^>]*>' + r'(?:(?:(?!).)*)', + re.DOTALL | re.IGNORECASE, +) + +_UNTERMINATED_REASONING_BLOCK_PATTERN = re.compile( + rf'(?:^|\n)[ \t]*<(?:{"|".join(_REASONING_TAG_NAMES)})\b[^>]*>.*$', + re.DOTALL | re.IGNORECASE, +) + +_ORPHAN_REASONING_TAG_PATTERN = re.compile( + rf'\s*', + re.IGNORECASE, +) + +_STRAY_TOOL_CALL_CLOSER_PATTERN = re.compile( + rf'\s*', + re.IGNORECASE, +) + + def _ra(): """Lazy ``run_agent`` reference for test-patch routing.""" import run_agent @@ -151,7 +190,7 @@ def convert_to_trajectory_format(agent, messages: List[Dict[str, Any]], user_que except json.JSONDecodeError: # This shouldn't happen since we validate and retry during conversation, # but if it does, log warning and use empty dict - logger.warning(f"Unexpected invalid JSON in trajectory conversion: {tool_call['function']['arguments'][:100]}") + logger.warning("Unexpected invalid JSON in trajectory conversion: %s", tool_call['function']['arguments'][:100]) arguments = {} tool_call_json = { @@ -249,12 +288,42 @@ def sanitize_tool_call_arguments( *, logger=None, session_id: str = None, + cursor: Optional[dict] = None, ) -> int: - """Repair corrupted assistant tool-call argument JSON in-place.""" + """Repair corrupted assistant tool-call argument JSON in-place. + + ``cursor`` (optional) is a caller-owned dict used to skip re-validating + messages already validated on a previous call. It stores, under + ``"prefix"``, the exact message *objects* (strong references) validated + last time, in order. On the next call, the longest contiguous prefix of + ``messages`` whose objects are ``is``-identical to the stored prefix is + skipped; scanning starts at the first divergence (conservative: any + reordering, truncation, compression rewrite, or mid-list insertion breaks + identity at that index and everything from there is re-scanned). + + Safety argument for skipping: a message in the matched prefix was fully + scanned before — every tool_call argument was either already valid JSON + or was rewritten to ``"{}"`` (valid). The only code paths that mutate + ``function["arguments"]`` on live history dicts between calls are the + surrogate / non-ASCII sanitizers, which substitute characters *inside* + JSON string values and cannot invalidate JSON syntax. Compression, + repair, undo, and steer paths replace or reorder message dicts, which + breaks the identity match and forces a re-scan. Holding strong + references (the objects themselves, not ``id()``s) makes address reuse + aliasing (#50372-style) impossible. + """ log = logger or logging.getLogger(__name__) if not isinstance(messages, list): return 0 + start_index = 0 + if cursor is not None: + prev_prefix = cursor.get("prefix") + if isinstance(prev_prefix, list): + limit = min(len(prev_prefix), len(messages)) + while start_index < limit and messages[start_index] is prev_prefix[start_index]: + start_index += 1 + repaired = 0 marker = _ra().AIAgent._TOOL_CALL_ARGUMENTS_CORRUPTION_MARKER @@ -275,7 +344,7 @@ def _prepend_marker(tool_msg: dict) -> None: existing_text = str(existing) tool_msg["content"] = f"{marker}\n{existing_text}" - message_index = 0 + message_index = start_index while message_index < len(messages): msg = messages[message_index] if not isinstance(msg, dict) or msg.get("role") != "assistant": @@ -356,6 +425,12 @@ def _prepend_marker(tool_msg: dict) -> None: message_index += 1 + if cursor is not None: + # Strong references to the exact objects validated this call, in + # order. Any future divergence (compression, undo, repair, steer) + # breaks identity at the divergent index and re-scans from there. + cursor["prefix"] = messages[:] + return repaired @@ -790,66 +865,54 @@ def strip_think_blocks(agent, content: str) -> str: # 1. Closed tag pairs — case-insensitive for all variants so # mixed-case tags (, ) don't slip through to # the unterminated-tag pass and take trailing content with them. - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - content = re.sub(r'.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + for _pattern in _REASONING_BLOCK_PATTERNS: + content = _pattern.sub('', content) # 1b. Tool-call XML blocks (openclaw/openclaw#67318). Handle the # generic tag names first — they have no attribute gating since # a literal in prose is already vanishingly rare. - for _tc_name in ("tool_call", "tool_calls", "tool_result", - "function_call", "function_calls"): - content = re.sub( - rf'<{_tc_name}\b[^>]*>.*?', - '', - content, - flags=re.DOTALL | re.IGNORECASE, - ) + for _pattern in _TOOL_CALL_BLOCK_PATTERNS: + content = _pattern.sub('', content) # 1c. ... — Gemma-style standalone # tool call. Only strip when the tag sits at a block boundary # (start of text, after a newline, or after sentence-ending # punctuation) AND carries a name="..." attribute. This keeps # prose mentions like "Use to declare" safe. - content = re.sub( - r'(?:(?<=^)|(?<=[\n\r.!?:]))[ \t]*' - r']*\bname\s*=[^>]*>' - r'(?:(?:(?!).)*)', - '', - content, - flags=re.DOTALL | re.IGNORECASE, - ) + content = _NAMED_FUNCTION_BLOCK_PATTERN.sub('', content) # 2. Unterminated reasoning block — open tag at a block boundary # (start of text, or after a newline) with no matching close. # Strip from the tag to end of string. Fixes #8878 / #9568 # (MiniMax M2.7 leaking raw reasoning into assistant content). - content = re.sub( - r'(?:^|\n)[ \t]*<(?:think|thinking|reasoning|thought|REASONING_SCRATCHPAD)\b[^>]*>.*$', - '', - content, - flags=re.DOTALL | re.IGNORECASE, - ) + content = _UNTERMINATED_REASONING_BLOCK_PATTERN.sub('', content) # 3. Stray orphan open/close tags that slipped through. - content = re.sub( - r'\s*', - '', - content, - flags=re.IGNORECASE, - ) + content = _ORPHAN_REASONING_TAG_PATTERN.sub('', content) # 3b. Stray tool-call closers. (We do NOT strip bare or # unterminated because a truncated tail # during streaming may still be valuable to the user; matches # OpenClaw's intentional asymmetry.) - content = re.sub( - r'\s*', - '', - content, - flags=re.IGNORECASE, - ) + content = _STRAY_TOOL_CALL_CLOSER_PATTERN.sub('', content) return content +def sync_credential_pool_entry_id(agent) -> None: + """Rebind ``agent._credential_pool_entry_id`` from the current pool + key. + + OAuth refreshes can replace the runtime token before a failed request is + recovered, so the mutable API-key value alone cannot reliably attribute + the failure to its source entry. This resolves the stable pool-entry ID + for the agent's current ``api_key`` and clears it when no pool is bound. + """ + pool = getattr(agent, "_credential_pool", None) + try: + agent._credential_pool_entry_id = ( + pool.entry_id_for_api_key(getattr(agent, "api_key", None)) + if pool is not None + else None + ) + except Exception: + agent._credential_pool_entry_id = None + + def recover_with_credential_pool( agent, *, @@ -934,10 +997,30 @@ def recover_with_credential_pool( # failing entry exactly; fall back to current()'s key only when the agent # carries no key at all. _api_key_hint = getattr(agent, "api_key", None) or None + _raw_credential_id = getattr(agent, "_credential_pool_entry_id", None) + _credential_id = ( + _raw_credential_id + if isinstance(_raw_credential_id, str) and _raw_credential_id + else None + ) if not _api_key_hint: _cur = pool.current() if _cur: _api_key_hint = getattr(_cur, "runtime_api_key", None) + if not _credential_id: + _current_id = getattr(_cur, "id", None) + if isinstance(_current_id, str) and _current_id: + _credential_id = _current_id + + def _rotate_failed_credential(rotate_status: int): + kwargs = { + "status_code": rotate_status, + "error_context": error_context, + "api_key_hint": _api_key_hint, + } + if _credential_id: + kwargs["credential_id"] = _credential_id + return pool.mark_exhausted_and_rotate(**kwargs) effective_reason = classified_reason if effective_reason is None: @@ -972,11 +1055,7 @@ def recover_with_credential_pool( # Runtime credentials can be resolved by a separate pool instance, # leaving this recovery pool without ``current_id``. Match the key # that actually failed instead of quarantining a different account. - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (billing) — rotated to pool entry %s", @@ -995,8 +1074,13 @@ def recover_with_credential_pool( # Prefer the entry matching the failing key over the shared current() # pointer, for the same attribution reason as above. current_entry = None - if _api_key_hint: + if _credential_id: current_entry = next( + (e for e in pool.entries() if e.id == _credential_id), + None, + ) + if _api_key_hint: + current_entry = current_entry or next( (e for e in pool.entries() if e.runtime_api_key == _api_key_hint), None, ) @@ -1009,11 +1093,7 @@ def recover_with_credential_pool( current_last_status, ) rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s", @@ -1037,11 +1117,7 @@ def recover_with_credential_pool( if not has_retried_429 and not usage_limit_reached: return False, True rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (rate limit) — rotated to pool entry %s", @@ -1113,7 +1189,10 @@ def recover_with_credential_pool( # the shared pointer can reference a different, healthy entry, and # refreshing it would consume that entry's single-use refresh token # (or mark it exhausted on failure) for a failure it never had. - refreshed = pool.try_refresh_matching(api_key_hint=_api_key_hint) + refresh_kwargs = {"api_key_hint": _api_key_hint} + if _credential_id: + refresh_kwargs["credential_id"] = _credential_id + refreshed = pool.try_refresh_matching(**refresh_kwargs) if refreshed is not None: # ``try_refresh_matching()`` re-mints a fresh OAuth token and reports # success even when the upstream keeps rejecting it — a single-entry @@ -1139,17 +1218,13 @@ def recover_with_credential_pool( refreshed_id, ) return False, has_retried_429 - _ra().logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") + _ra().logger.info("Credential auth failure — refreshed pool entry %s", getattr(refreshed, 'id', '?')) agent._swap_credential(refreshed) return True, has_retried_429 # Refresh failed — rotate to next credential instead of giving up. # The failed entry is already marked exhausted by the refresh attempt. rotate_status = status_code if status_code is not None else 401 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - api_key_hint=_api_key_hint, - ) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (auth refresh failed) — rotated to pool entry %s", @@ -1190,15 +1265,29 @@ def try_recover_primary_transport( if agent._is_openrouter_url(): return False provider_lower = (agent.provider or "").strip().lower() - if provider_lower in {"nous", "nous-research"}: + # Portal OpenAI-wire traffic still rides aggregator retry infra, so one + # more rebuilt OpenAI client won't help. Portal Claude on the native + # Messages route holds a local Anthropic SDK client whose connection + # pool *does* need the rebuild every other anthropic_messages provider + # already gets — don't blanket-skip the dual-wire path. + if ( + provider_lower in {"nous", "nous-portal", "nousresearch"} + and getattr(agent, "api_mode", None) != "anthropic_messages" + ): return False try: - # Close existing client to release stale connections + # Retire the existing client to release stale connections. #70773: + # never hard-close the shared client here — this runs on the + # conversation-loop thread while workers from stale-killed streaming + # attempts may still be unwinding their SSL BIOs on the old pool. + # ``_retire_shared_openai_client`` shuts the sockets down (FD-safe + # from any thread) and defers the FD release to GC, which cannot + # complete until every borrowing thread has unwound. if getattr(agent, "client", None) is not None: try: - agent._close_openai_client( - agent.client, reason="primary_recovery", shared=True, + agent._retire_shared_openai_client( + agent.client, reason="primary_recovery", ) except Exception: pass @@ -1375,6 +1464,64 @@ def restore_primary_runtime(agent) -> bool: if getattr(agent, "_rate_limited_until", 0) > time.monotonic(): return False # primary still in rate-limit cooldown, stay on fallback + # ── Reset-aware gate ── + # The 60s ``_rate_limited_until`` cooldown covers transient rate limits, + # but subscription-style providers (Claude Pro/Max 5-hour windows, ChatGPT + # weekly limits) report reset times hours or days away. The credential + # pool already stores those timestamps (``last_error_reset_at``); until + # the earliest one elapses, every restore attempt is a *guaranteed* + # failure that costs two prompt-cache invalidations per turn (switch to + # primary, fail, switch back to fallback) and re-marshals the full + # context each way. Skip the restore while the pool says nobody can + # serve, and come back the moment the reset time passes. + # + # Fail-open by design: any error (unreadable auth store, legacy pool + # adapter without ``next_available_at``) falls through to the existing + # every-turn retry. A pool with no reset info returns ``None`` and also + # falls through — this gate only ever *adds* skips for provably + # limited windows, so recovery can never be later than it is today. + # + # When the attached pool belongs to the fallback provider (cross-provider + # fallback rebinds it), the primary pool is loaded here and handed to the + # pool-rebind block below via ``prefetched_primary_pool`` so the load + # happens at most once per restore. + prefetched_primary_pool = None + try: + primary_provider = str( + (agent._primary_runtime or {}).get("provider") or "" + ).strip().lower() + pool = getattr(agent, "_credential_pool", None) + if not credential_pool_matches_provider( + pool, + primary_provider, + base_url=str((agent._primary_runtime or {}).get("base_url") or ""), + ): + from agent.credential_pool import load_pool + + prefetched_primary_pool = ( + load_pool(primary_provider) if primary_provider else None + ) + pool = prefetched_primary_pool + next_at = getattr(pool, "next_available_at", lambda: None)() + if next_at is not None and next_at > time.time(): + if not getattr(agent, "_restore_wait_logged", False): + agent._restore_wait_logged = True + logger.info( + "Primary %s rate-limited until %s; staying on fallback " + "%s/%s until the reset elapses", + primary_provider or "?", + datetime.fromtimestamp(next_at).isoformat(timespec="seconds"), + agent.provider, + agent.model, + ) + return False + except Exception: + logger.debug( + "Reset-aware restore gate failed; falling back to per-turn retry", + exc_info=True, + ) + agent._restore_wait_logged = False + rt = agent._primary_runtime try: # ── Core runtime state ── @@ -1394,6 +1541,12 @@ def restore_primary_runtime(agent) -> bool: "use_native_cache_layout", agent.api_mode == "anthropic_messages" and agent.provider == "anthropic", ) + # If the operator has disabled caching via config (cache_ttl is + # falsy → _cache_disabled flag is set), the disable must survive + # runtime snapshot restoration (#33555). + if getattr(agent, "_cache_disabled", False): + agent._use_prompt_caching = False + agent._use_native_cache_layout = False # ── Rebuild client for the primary provider ── if agent.provider == "moa": @@ -1460,10 +1613,16 @@ def restore_primary_runtime(agent) -> bool: pool_matches_primary = False if pool is not None and pool_provider and not pool_matches_primary: agent._credential_pool = None + agent._credential_pool_entry_id = None try: - from agent.credential_pool import load_pool + if prefetched_primary_pool is not None: + # Reuse the pool the reset-aware gate already loaded for + # this restore — avoids a second disk read of auth.json. + agent._credential_pool = prefetched_primary_pool + else: + from agent.credential_pool import load_pool - agent._credential_pool = load_pool(primary_provider) + agent._credential_pool = load_pool(primary_provider) except Exception as exc: logger.warning( "Restore could not reload primary credential pool for %s: %s", @@ -1479,6 +1638,7 @@ def restore_primary_runtime(agent) -> bool: # the pool for its current best entry and swap the live credential in. # When the pool is absent, empty, or the entry has no usable key, we # keep the snapshot key (the existing behavior). Fixes #25205. + agent._credential_pool_entry_id = None pool = getattr(agent, "_credential_pool", None) if pool is not None and pool.has_available(): entry = pool.select() @@ -1746,11 +1906,148 @@ def dump_api_request_debug( return dump_file except Exception as dump_error: if agent.verbose_logging: - logger.warning(f"Failed to dump API request debug payload: {dump_error}") + logger.warning("Failed to dump API request debug payload: %s", dump_error) return None +def _direct_native_anthropic_tool_cache_capability( + agent, + *, + provider: Optional[str] = None, + base_url: Optional[str] = None, + api_mode: Optional[str] = None, + model: Optional[str] = None, +) -> bool: + """Return whether this resolved destination accepts native tool markers.""" + eff_base_url = base_url if base_url is not None else (agent.base_url or "") + eff_api_mode = api_mode if api_mode is not None else (agent.api_mode or "") + return ( + eff_api_mode == "anthropic_messages" + and base_url_hostname(eff_base_url) == "api.anthropic.com" + ) + + +def cache_ttl_means_disabled(ttl: Any) -> bool: + """Return True when a ``prompt_caching.cache_ttl`` value means caching off. + + Single source of truth for the disable-synonym detection shared by + ``agent_init`` (live-agent ``_cache_disabled`` flag) and the stub policy + paths below. Keeping one predicate prevents the two sites from drifting + (a synonym added in only one place would recreate #76085). + + Unknown values (e.g. ``"2h"``, integers) are NOT a disable — callers keep + caching enabled with the default TTL, matching ``agent_init``. + """ + if ttl in ("5m", "1h"): + return False + if ttl is False or ttl is None: + return True + return str(ttl).lower() in ("off", "false", "disabled", "no", "none") + + +def prompt_caching_disabled_from_config() -> bool: + """Return True when ``prompt_caching.cache_ttl`` is configured as off. + + Same disable detection as ``agent_init`` (via ``cache_ttl_means_disabled``) + so stub-based policy paths (MoA slot decoration, auxiliary fallback + replan) honor the same config contract without holding a live + ``AIAgent`` (#76085 / #33555). + """ + try: + from hermes_cli.config import load_config_readonly + + pc_cfg = load_config_readonly().get("prompt_caching", {}) or {} + ttl = pc_cfg.get("cache_ttl", "5m") + except Exception: + return False + return cache_ttl_means_disabled(ttl) + + +def blank_cache_policy_stub(cache_disabled: Optional[bool] = None): + """Build the destination-identity-blank stub for ``anthropic_prompt_cache_policy``. + + Single sanctioned constructor for that stub. Callers that resolve cache + policy against a destination identified out-of-band (not a live + ``AIAgent``) must go through here so ``_cache_disabled`` is never left + off a hand-rolled ``SimpleNamespace`` (#76085). + + When ``cache_disabled`` is omitted, falls back to the global config so + stub paths without an agent snapshot still honor an operator disable. + """ + from types import SimpleNamespace + + if cache_disabled is None: + cache_disabled = prompt_caching_disabled_from_config() + return SimpleNamespace( + provider="", + base_url="", + api_mode="", + model="", + _cache_disabled=bool(cache_disabled), + ) + + +def plan_cache_sections_for_destination( + messages: list, + tools: Optional[list], + *, + provider: str, + base_url: str, + api_mode: str, + model: str, + cache_disabled: Optional[bool] = None, +) -> Tuple[list, list]: + """Plan request-local cache sections for one resolved destination. + + Shared core of the synchronous acting-aggregator (MoA) and auxiliary + fallback senders: resolve the cache policy for the destination's real + provider/base_url/api_mode/model, then either return stripped canonical + copies (non-caching route) or a :func:`build_prompt_cache_plan` layout + (caching route, with the direct-native tool marker when the destination + is api.anthropic.com on the Messages wire). + + Never mutates ``messages`` or ``tools`` — both return values are + request-local copies. + + ``cache_disabled`` threads the operator's ``prompt_caching.cache_ttl`` + disable into the blank policy stub. When omitted, the live config is + consulted so MoA/auxiliary paths cannot re-enable markers after the + user turned caching off (#76085). + """ + from agent.prompt_caching import ( + build_prompt_cache_plan, + strip_anthropic_cache_control, + strip_anthropic_tool_cache_control, + ) + + stub = blank_cache_policy_stub(cache_disabled) + should_cache, native_layout = anthropic_prompt_cache_policy( + stub, + provider=provider, + base_url=base_url, + api_mode=api_mode, + model=model, + ) + if not should_cache: + canonical_messages = copy.deepcopy(messages or []) + strip_anthropic_cache_control(canonical_messages) + return canonical_messages, strip_anthropic_tool_cache_control(tools) + plan = build_prompt_cache_plan( + messages, + tools, + native_anthropic=native_layout, + direct_native_tool_cache=_direct_native_anthropic_tool_cache_capability( + stub, + provider=provider, + base_url=base_url, + api_mode=api_mode, + model=model, + ), + ) + return plan.messages, plan.tools + + def anthropic_prompt_cache_policy( agent, *, @@ -1783,7 +2080,19 @@ def anthropic_prompt_cache_policy( pi #3393 documented this for opencode-go Qwen. Without markers these providers serve zero cache hits, re-billing the full prompt on every turn. + + If the operator has set ``prompt_caching.cache_ttl`` to a falsy value + (``false``, ``null``, ``"off"``, etc.) in config.yaml, prompt caching + is fully disabled — this early return ensures the disable survives + ``/model`` switches, fallback re-derivation, and runtime snapshot + restoration (#33555). We check ``"_cache_disabled"`` (set by + init_agent when the disable is detected) rather than ``_cache_ttl`` + directly, because ``_cache_ttl`` is not yet set when the policy runs + during the initial ``init_agent`` call. """ + if getattr(agent, "_cache_disabled", False): + return (False, False) + eff_provider = (provider if provider is not None else agent.provider) or "" eff_base_url = base_url if base_url is not None else (agent.base_url or "") eff_api_mode = api_mode if api_mode is not None else (agent.api_mode or "") @@ -1856,7 +2165,15 @@ def anthropic_prompt_cache_policy( if is_native_anthropic: return True, True - if (is_openrouter or is_nous_portal) and (is_claude or is_kimi): + # Envelope layout is an OpenAI-wire construct. Portal Claude on the native + # Messages route must fall through to the third-party anthropic_messages + # branch below, which emits inner-block cache_control breakpoints; the + # envelope form would be dropped and serve 0% cache hits. + if ( + (is_openrouter or is_nous_portal) + and (is_claude or is_kimi) + and not is_anthropic_wire + ): return True, False # Nous Portal Qwen (e.g. qwen3.6-plus) takes the same envelope-layout # cache_control path as Portal Claude. Portal proxies to OpenRouter @@ -1894,6 +2211,11 @@ def anthropic_prompt_cache_policy( # rewards them with real cache hits. Without this branch # qwen3.6-plus on opencode-go reports 0% cached tokens and burns # through the subscription on every turn. + # + # NOTE: DeepSeek models on OpenCode are intentionally excluded. + # OpenCode Zen's relay rejects the Anthropic-style content block + # format that cache markers produce (content becomes a block array + # instead of a plain string), causing HTTP 400 (#77217). model_is_qwen = "qwen" in model_lower provider_is_alibaba_family = provider_lower in { "opencode", "opencode-zen", "opencode-go", "alibaba", @@ -1991,6 +2313,31 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo # restore, request-scoped); auxiliary_client builds its own clients and keeps # SDK retries because it is NOT wrapped by the conversation loop. client_kwargs.setdefault("max_retries", 0) + # Defense-in-depth: guarantee Copilot requests carry the integration + # headers regardless of which build path we came through. The primary + # header wiring lives in `_apply_client_headers_for_base_url`, but two + # rebuild paths (`primary_recovery`, `restore_primary` in this module) + # reconstruct the client purely from a `_primary_runtime` snapshot and do + # NOT re-run that wiring. If the snapshot's client_kwargs ever lacks + # `default_headers` (older snapshot, header-less resolver result), the + # client goes out WITHOUT `Copilot-Integration-Id: vscode-chat`; the + # Copilot server then routes it to the "copilot-language-server" integrator + # whose model allowlist omits enterprise-only models (claude-opus-4.8) → + # HTTP 400 model_not_available_for_integrator on every turn. This chokepoint + # is the single place every primary OpenAI client passes through, so filling + # missing Copilot headers here closes the whole class. We only ADD missing + # keys — never override headers a caller deliberately set. + try: + if base_url_host_matches(str(client_kwargs.get("base_url", "")), "githubcopilot.com"): + from hermes_cli.models import copilot_default_headers + existing = dict(client_kwargs.get("default_headers") or {}) + existing_lower = {k.lower() for k in existing} + for hk, hv in copilot_default_headers().items(): + if hk.lower() not in existing_lower: + existing[hk] = hv + client_kwargs["default_headers"] = existing + except Exception: + _ra().logger.debug("Copilot default-header guard skipped", exc_info=True) # Uses the module-level `OpenAI` name, resolved lazily on first # access via __getattr__ below. Tests patch via `run_agent.OpenAI`. client = _ra().OpenAI(**client_kwargs) @@ -2020,8 +2367,11 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo from hermes_cli.providers import determine_api_mode # ── Determine api_mode if not provided ── + # Pass model so dual-wire providers (Nous Portal anthropic/* → Messages) + # resolve correctly; without it determine_api_mode falls back to the + # openai_chat overlay default. if not api_mode: - api_mode = determine_api_mode(new_provider, base_url) + api_mode = determine_api_mode(new_provider, base_url, model=new_model) # Defense-in-depth: ensure OpenCode base_url doesn't carry a trailing # /v1 into the anthropic_messages client, which would cause the SDK to @@ -2075,6 +2425,19 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # restore the original pool (issue #52727: pool reload is part of this # switch and must be reversible on rollback). _snapshot["_credential_pool"] = getattr(agent, "_credential_pool", _MISSING) + _snapshot["_credential_pool_entry_id"] = getattr( + agent, "_credential_pool_entry_id", _MISSING + ) + + def _restore_snapshot() -> None: + for _name, _value in _snapshot.items(): + if _value is _MISSING: + # Attribute did not exist before the swap — don't fabricate it. + continue + try: + setattr(agent, _name, _value) + except Exception: # noqa: BLE001 + pass try: # Clear the per-config context_length override so the new model's @@ -2131,6 +2494,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # A pool bound to the old provider is worse than no pool: the # recovery guard rejects it and every later 401/429 skips rotation. agent._credential_pool = None + agent._credential_pool_entry_id = None try: from agent.credential_pool import load_pool agent._credential_pool = load_pool(new_provider) @@ -2140,7 +2504,6 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo "continuing without pool rotation this turn", new_provider, _pool_exc, ) - # ── Build new client ── if (new_provider or "").strip().lower() == "moa": from agent.moa_loop import build_moa_facade @@ -2236,22 +2599,50 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo reason="switch_model", shared=True, ) + + sync_credential_pool_entry_id(agent) except Exception: # Rollback every mutated field to the pre-swap snapshot so the agent # is left consistent (old model + old provider + old client) and the # caller's exception handler can surface a meaningful warning. The # exception is re-raised; cli.py / gateway/run.py / tui_gateway catch # it and print "Agent swap failed; change applied to next session". - for _name, _value in _snapshot.items(): - if _value is _MISSING: - # Attribute did not exist before the swap — don't fabricate it. - continue - try: - setattr(agent, _name, _value) - except Exception: # noqa: BLE001 - pass + _restore_snapshot() raise + # ── LM Studio: preload before probing context length ── + _sm_custom_providers = None + try: + from hermes_cli.config import ( + get_compatible_custom_providers, + get_custom_provider_context_length, + load_config, + ) + + _sm_cfg = load_config() + _sm_custom_providers = get_compatible_custom_providers(_sm_cfg) + _destination_context_intent = get_custom_provider_context_length( + model=agent.model, + base_url=agent.base_url, + custom_providers=_sm_custom_providers, + ) + except Exception: + _destination_context_intent = None + agent._config_context_length = _destination_context_intent + _runtime_context_length = agent._ensure_lmstudio_runtime_loaded( + _destination_context_intent + ) + if agent._lmstudio_load_was_unverified(_runtime_context_length): + logger.warning( + "LM Studio model activation was rejected or completed without a " + "verifiable active context length during model switch; continuing " + "with configured context" + ) + _effective_context_length = agent._effective_lmstudio_context_length( + _destination_context_intent, + _runtime_context_length, + ) + # ── Re-evaluate prompt caching ── agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy( @@ -2262,22 +2653,15 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo ) ) - # ── LM Studio: preload before probing context length ── - agent._ensure_lmstudio_runtime_loaded() - # ── Update context compressor ── if hasattr(agent, "context_compressor") and agent.context_compressor: from agent.model_metadata import get_model_context_length - # Re-read custom_providers from live config so per-model - # context_length overrides are honored when switching to a - # custom provider mid-session (closes #15779). - _sm_custom_providers = None - try: - from hermes_cli.config import load_config, get_compatible_custom_providers - _sm_cfg = load_config() - _sm_custom_providers = get_compatible_custom_providers(_sm_cfg) - except Exception: - _sm_custom_providers = None + if _sm_custom_providers is None: + try: + from hermes_cli.config import get_compatible_custom_providers, load_config + _sm_custom_providers = get_compatible_custom_providers(load_config()) + except Exception: + _sm_custom_providers = None # ``agent.api_key`` may be a callable (Azure Foundry Entra ID # token provider). ``get_model_context_length`` expects a # string for its live-probe paths; for Foundry the context @@ -2289,7 +2673,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo base_url=agent.base_url, api_key=_ctx_api_key, provider=agent.provider, - config_context_length=getattr(agent, "_config_context_length", None), + config_context_length=_effective_context_length, custom_providers=_sm_custom_providers, ) agent.context_compressor.update_model( @@ -2412,7 +2796,8 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False, skip_tool_request_middleware: bool = False, - tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None) -> str: + tool_request_middleware_trace: Optional[List[Dict[str, Any]]] = None, + skip_tool_execution_middleware: bool = False) -> str: """Invoke a single tool and return the result string. No display logic. Handles both agent-level tools (todo, memory, etc.) and registry-dispatched @@ -2570,6 +2955,7 @@ def _execute(next_args: dict) -> Any: _clarify_tool( question=next_args.get("question", ""), choices=next_args.get("choices"), + multi_select=next_args.get("multi_select", False), callback=agent.clarify_callback, ), next_args, @@ -2590,8 +2976,7 @@ def _execute(next_args: dict) -> Any: return _finish_agent_tool(agent._dispatch_delegate_task(next_args), next_args) else: def _execute(next_args: dict) -> Any: - return _ra().handle_function_call( - function_name, next_args, effective_task_id, + dispatch_kwargs = dict( tool_call_id=tool_call_id, session_id=agent.session_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", @@ -2603,6 +2988,17 @@ def _execute(next_args: dict) -> Any: disabled_toolsets=getattr(agent, "disabled_toolsets", None), tool_request_middleware_trace=list(_tool_middleware_trace), ) + if skip_tool_execution_middleware: + dispatch_kwargs["skip_tool_execution_middleware"] = True + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + **dispatch_kwargs, + ) + + if skip_tool_execution_middleware: + return _execute(function_args) from hermes_cli.middleware import run_tool_execution_middleware @@ -2715,6 +3111,129 @@ def _strip_tool_suffix(s: str) -> str | None: +# Placeholder substituted for an empty non-final message that would otherwise +# make the provider reject the whole request. Kept identical to the stub- +# creation placeholder in chat_completion_helpers so a healed transcript reads +# consistently whether the empty turn was caught at write time or send time. +_INTERRUPTED_PLACEHOLDER = "[response interrupted]" + + +def _msg_has_payload(msg: Dict[str, Any]) -> bool: + """True if ``msg`` carries anything the API treats as non-empty content. + + Covers string content, non-empty multimodal content lists, tool_calls, + tool_call_id linkage (tool results), and reasoning payloads. Mirrors the + emptiness checks used by ``AIAgent._is_thinking_only_assistant`` but is + role-agnostic so it can vet user/assistant/tool turns uniformly. + """ + content = msg.get("content") + if isinstance(content, str): + if content.strip(): + return True + elif isinstance(content, list): + for block in content: + if isinstance(block, dict): + # any typed block (text/image/tool_use/document/...) counts, + # as long as a text block is not itself blank + if block.get("type") == "text": + if isinstance(block.get("text"), str) and block["text"].strip(): + return True + continue + return True + elif block: + return True + elif content not in (None, ""): + return True + # Structural payloads that make an "empty-content" message still valid. + if msg.get("tool_calls"): + return True + if isinstance(msg.get("reasoning_content"), str) and msg["reasoning_content"].strip(): + return True + if msg.get("reasoning") or msg.get("reasoning_details"): + return True + # Codex Responses item carriers: a commentary-phase assistant turn + # persists with content:"" by DESIGN — its text lives in + # ``codex_message_items`` (delivered via the interim callback) and the + # structured items are replayed for prefix-cache hits. Same for + # ``codex_reasoning_items``. These turns are never wire-empty on any + # api_mode: the codex transport replays the items, and the + # chat-completions transport strips the carriers only after this repair + # pass has already run. Treat them as payload so the repair never + # rewrites a designed-empty codex turn (July 2026: a write-time pad that + # ignored this broke codex commentary replay in CI). + if msg.get("codex_message_items") or msg.get("codex_reasoning_items"): + return True + return False + + +def repair_empty_non_final_messages( + messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Heal empty-content non-final messages before they reach the provider. + + Root-cause context: a stream that dies with 0 recovered characters (peer + reset, stall-kill) could persist an assistant turn with ``content=None`` + and no tool_calls. The Anthropic message schema — and the litellm/Bedrock + proxies in front of it — reject ANY request whose transcript contains an + empty non-final message: + + "all messages must have non-empty content except for the optional + final assistant message" (HTTP 400 INVALID_REQUEST_BODY) + + Once such a message lands mid-transcript it poisons EVERY subsequent turn + of that session until it scrolls out of context. The write-time guard in + ``chat_completion_helpers`` stops NEW stubs, but sessions already carrying + one (persisted before the guard, or fed in from a host history) stay stuck + and previously needed a manual DB edit + gateway restart to recover. + + This pass is the self-healing counterpart: it runs unconditionally on the + per-call ``api_messages`` copy, so a poisoned transcript repairs itself + IN MEMORY on the very next send — no restart, no DB surgery. The final + message is left untouched (an empty final assistant turn is legal). The + stored conversation history is never mutated; only the wire copy is + repaired, so the UI/session trace stays faithful. + + Repair strategy is substitution, not deletion: dropping a mid-transcript + turn can break role alternation and tool-call pairing, whereas an honest + minimal placeholder keeps the sequence intact and reads correctly as an + interrupted turn on replay. + """ + if not messages or len(messages) < 2: + return messages + + repaired: List[Dict[str, Any]] = [] + healed = 0 + last_idx = len(messages) - 1 + for idx, msg in enumerate(messages): + if ( + idx != last_idx + and isinstance(msg, dict) + # tool results are validated by their own orphan/pairing pass; an + # empty tool result is a separate (and rarer) concern. + and msg.get("role") in ("assistant", "user") + and not _msg_has_payload(msg) + ): + # Shallow-copy so stored history / prompt caching stays byte-stable. + fixed = dict(msg) + fixed["content"] = _INTERRUPTED_PLACEHOLDER + repaired.append(fixed) + healed += 1 + else: + repaired.append(msg) + + if healed: + _ra().logger.warning( + "Pre-call sanitizer: healed %d empty non-final message(s) by " + "substituting placeholder content — an empty-content turn was in " + "the transcript and would 400 the request ('messages must have " + "non-empty content' / INVALID_REQUEST_BODY). Self-recovering the " + "poisoned transcript in memory; no restart needed.", + healed, + ) + return repaired + return messages + + def sanitize_api_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Fix orphaned tool_call / tool_result pairs before every LLM call. @@ -2735,6 +3254,15 @@ def sanitize_api_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any] filtered.append(msg) messages = filtered + # --- Heal empty-content non-final messages (self-recovery) --- + # A dead stream can leave an empty assistant stub (or an empty user turn) + # mid-transcript; the provider then 400s EVERY subsequent request until it + # scrolls out. Repair it here, on the per-call copy, so a poisoned session + # recovers itself in memory on the next send — no restart, no DB edit. + # Done first so a substituted turn participates normally in the tool-pair + # and dedup passes below. + messages = repair_empty_non_final_messages(messages) + # --- Drop empty / malformed tool_calls arrays on assistant messages --- # An assistant message carrying ``tool_calls: []`` (an empty array) — or a # non-list value under the key — is semantically identical to an assistant @@ -3059,89 +3587,17 @@ def intent_ack_continuation_enabled(agent) -> bool: def copy_reasoning_content_for_api(agent, source_msg: dict, api_msg: dict) -> None: - """Copy provider-facing reasoning fields onto an API replay message.""" - if source_msg.get("role") != "assistant": - return - - needs_thinking_pad = agent._needs_thinking_reasoning_pad() + """Copy provider-facing reasoning fields onto an API replay message. - # 1. Explicit reasoning_content already set. - # - # When the active provider enforces the thinking-mode echo-back - # (DeepSeek / Kimi / MiMo), preserve it verbatim — that includes their - # own space-placeholder written at creation time and any valid reasoning - # from the same provider. Sessions persisted BEFORE #17341 have - # empty-string placeholders pinned at creation time; DeepSeek V4 Pro - # rejects those with HTTP 400, so upgrade "" → " " on replay. - # - # When the active provider does NOT enforce echo-back, strip the field - # entirely. Strict OpenAI-compatible providers (Mistral, Cerebras, Groq, - # SambaNova, …) reject ANY reasoning_content key in input messages with - # HTTP 400/422 ("Extra inputs are not permitted"), even an empty string - # or a single-space pad. This is the cross-provider fallback case: a - # reasoning primary (DeepSeek/Kimi/MiMo) pads history with " ", then a - # fallback to a strict provider replays that pad and 422s. Stripping - # here covers the rebuild path; reapply_reasoning_echo_for_provider() - # covers the already-built api_messages path. Refs #45655. - existing = source_msg.get("reasoning_content") - if isinstance(existing, str): - if not needs_thinking_pad: - api_msg.pop("reasoning_content", None) - elif existing == "": - api_msg["reasoning_content"] = " " - else: - api_msg["reasoning_content"] = existing - return - - # 2. Cross-provider poisoned history (#15748): on DeepSeek/Kimi, - # if the source turn has tool_calls AND a 'reasoning' field but no - # 'reasoning_content' key, the 'reasoning' text was written by a - # prior provider (e.g. MiniMax) — DeepSeek's own _build_assistant_message - # pins reasoning_content at creation time for tool-call turns, so the - # shape (reasoning set, reasoning_content absent, tool_calls present) - # is unreachable from same-provider DeepSeek history after this fix. - # Inject a single space to satisfy the API without leaking another - # provider's chain of thought to DeepSeek/Kimi. Space (not "") - # because DeepSeek V4 Pro rejects empty-string reasoning_content - # in thinking mode (refs #17341). - normalized_reasoning = source_msg.get("reasoning") - if ( - needs_thinking_pad - and source_msg.get("tool_calls") - and isinstance(normalized_reasoning, str) - and normalized_reasoning - ): - api_msg["reasoning_content"] = " " - return - - # 3. Healthy session: promote 'reasoning' field to 'reasoning_content' - # for providers that use the internal 'reasoning' key. - # This must happen before the unconditional empty-string fallback so - # genuine reasoning content is not overwritten (#15812 regression in - # PR #15478). Only promote for providers that enforce echo-back — - # strict providers reject the field (refs #45655). - if isinstance(normalized_reasoning, str) and normalized_reasoning: - if needs_thinking_pad: - api_msg["reasoning_content"] = normalized_reasoning - else: - api_msg.pop("reasoning_content", None) - return - - # 4. DeepSeek / Kimi thinking mode: all assistant messages need - # reasoning_content. Inject a single space to satisfy the provider's - # requirement when no explicit reasoning content is present. Covers - # both tool-call turns (already-poisoned history with no reasoning - # at all) and plain text turns. Space (not "") because DeepSeek V4 - # Pro tightened validation and rejects empty string with HTTP 400 - # ("The reasoning content in the thinking mode must be passed back - # to the API"). Refs #17341. - if needs_thinking_pad: - api_msg["reasoning_content"] = " " - return + Forwarder — the strip-vs-repad POLICY is owned by + ``agent.message_sanitization.apply_reasoning_content_policy`` (audit F4); + this only supplies the agent's cached provider-direction flag. + """ + from agent.message_sanitization import apply_reasoning_content_policy - # 5. reasoning_content was present but not a string (e.g. None after - # context compaction). Don't pass null to the API. - api_msg.pop("reasoning_content", None) + apply_reasoning_content_policy( + source_msg, api_msg, agent._needs_thinking_reasoning_pad() + ) def reapply_reasoning_echo_for_provider(agent, api_messages: list) -> int: @@ -3173,25 +3629,74 @@ def reapply_reasoning_echo_for_provider(agent, api_messages: list) -> int: Returns the number of assistant turns whose reasoning_content was added or removed. """ - needs_pad = agent._needs_thinking_reasoning_pad() - changed = 0 - for api_msg in api_messages: - if api_msg.get("role") != "assistant": + from agent.message_sanitization import reapply_reasoning_echo + + return reapply_reasoning_echo( + api_messages, agent._needs_thinking_reasoning_pad() + ) + + +def _iter_httpx_pool_objects(http_client: Any): + """Yield httpcore pool objects reachable from an httpx client. + + Hermes' keepalive client (#10324 / ``_build_keepalive_http_client``) and + any ``HTTP(S)_PROXY`` configuration put live connections on *mounted* + transports (``client._mounts``), not only on the default + ``client._transport``. Walking the default transport alone makes + ``force_close_tcp_sockets`` return 0 while a stream is still mid-recv — + the interrupt logs success and the provider keeps burning the slot + (#72975). + """ + seen_pools: set[int] = set() + + def _emit(pool: Any): + if pool is None: + return + marker = id(pool) + if marker in seen_pools: + return + seen_pools.add(marker) + yield pool + + def _pools_for_transport(transport: Any): + if transport is None: + return + # Normal httpx.HTTPTransport / HTTPProxy-as-transport: connections + # live under ``_pool``. HTTPProxy itself *is* a ConnectionPool and + # may be mounted directly — then ``_connections`` is on the + # transport. + pool = getattr(transport, "_pool", None) + if pool is not None: + yield from _emit(pool) + return + if getattr(transport, "_connections", None) is not None: + yield from _emit(transport) + + try: + yield from _pools_for_transport(getattr(http_client, "_transport", None)) + mounts = getattr(http_client, "_mounts", None) or {} + for _pattern, mounted in list(mounts.items()): + yield from _pools_for_transport(mounted) + except Exception: + return + + +def _connection_candidates(conn: Any): + """Walk nested ``_connection`` wrappers (proxy tunnel → HTTP11/2).""" + seen: set[int] = set() + stack = [conn] + while stack: + candidate = stack.pop() + if candidate is None: continue - if needs_pad: - if api_msg.get("reasoning_content"): - continue - copy_reasoning_content_for_api(agent, api_msg, api_msg) - if api_msg.get("reasoning_content"): - changed += 1 - else: - # Strict provider — strip any stale reasoning_content pad left - # over from a reasoning primary so the fallback request doesn't - # 400/422 on it. - if "reasoning_content" in api_msg: - api_msg.pop("reasoning_content", None) - changed += 1 - return changed + marker = id(candidate) + if marker in seen: + continue + seen.add(marker) + yield candidate + inner = getattr(candidate, "_connection", None) + if inner is not None and id(inner) not in seen: + stack.append(inner) def _iter_pool_sockets(client: Any): @@ -3199,70 +3704,71 @@ def _iter_pool_sockets(client: Any): httpcore 1.x stores the concrete HTTP11/HTTP2 connection under ``conn._connection``; older versions exposed stream attributes directly - on the pool entry. Keep the traversal defensive because these are private - transport internals and vary across httpx/httpcore releases. + on the pool entry. Proxy tunnels wrap another layer + (``TunnelHTTPConnection`` / ``ForwardHTTPConnection``). Keep the + traversal defensive because these are private transport internals and + vary across httpx/httpcore releases. + + Also walks ``httpx`` mount transports — see ``_iter_httpx_pool_objects``. """ try: http_client = getattr(client, "_client", None) if http_client is None: - return - transport = getattr(http_client, "_transport", None) - if transport is None: - return - pool = getattr(transport, "_pool", None) - if pool is None: - return + # Some SDK wrappers *are* the httpx client (or expose the pool + # directly). Fall through so mount-aware discovery still runs. + http_client = client + pools = list(_iter_httpx_pool_objects(http_client)) + except Exception: + return + + if not pools: + return + + seen: set[int] = set() + for pool in pools: connections = ( getattr(pool, "_connections", None) or getattr(pool, "_pool", None) or [] ) - except Exception: - return - - seen: set[int] = set() - for conn in list(connections): - candidates = [conn] - inner = getattr(conn, "_connection", None) - if inner is not None: - candidates.append(inner) - for candidate in candidates: - stream = ( - getattr(candidate, "_network_stream", None) - or getattr(candidate, "_stream", None) - ) - if stream is None: - continue - sock = getattr(stream, "_sock", None) - if sock is None: - get_extra_info = getattr(stream, "get_extra_info", None) - if callable(get_extra_info): - try: - sock = get_extra_info("socket") - except Exception: - sock = None - if sock is None: - wrapped = getattr(stream, "stream", None) - if wrapped is not None: - sock = getattr(wrapped, "_sock", None) - if sock is None: - # anyio-backed streams expose the raw socket through - # SocketAttribute.raw_socket when available. - wrapped = getattr(stream, "_stream", None) - extra = getattr(wrapped, "extra", None) - if callable(extra): - try: - from anyio.abc import SocketAttribute - sock = extra(SocketAttribute.raw_socket) - except Exception: - sock = None - if sock is None: - continue - marker = id(sock) - if marker in seen: - continue - seen.add(marker) - yield sock + for conn in list(connections): + for candidate in _connection_candidates(conn): + stream = ( + getattr(candidate, "_network_stream", None) + or getattr(candidate, "_stream", None) + ) + if stream is None: + continue + sock = getattr(stream, "_sock", None) + if sock is None: + get_extra_info = getattr(stream, "get_extra_info", None) + if callable(get_extra_info): + try: + sock = get_extra_info("socket") + except Exception: + sock = None + if sock is None: + wrapped = getattr(stream, "stream", None) + if wrapped is not None: + sock = getattr(wrapped, "_sock", None) + if sock is None: + # anyio-backed streams expose the raw socket through + # SocketAttribute.raw_socket when available. + wrapped = getattr(stream, "_stream", None) + extra = getattr(wrapped, "extra", None) + if callable(extra): + try: + from anyio.abc import SocketAttribute + sock = extra(SocketAttribute.raw_socket) + except Exception: + sock = None + if sock is None: + continue + marker = id(sock) + if marker in seen: + continue + seen.add(marker) + yield sock def cleanup_dead_connections(agent) -> bool: @@ -3523,6 +4029,9 @@ def force_close_tcp_sockets(client: Any) -> int: "restore_primary_runtime", "extract_reasoning", "dump_api_request_debug", + "prompt_caching_disabled_from_config", + "blank_cache_policy_stub", + "plan_cache_sections_for_destination", "anthropic_prompt_cache_policy", "create_openai_client", "switch_model", diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 38431a8c1f5a..9ba37bd39b31 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -23,7 +23,19 @@ from hermes_constants import get_hermes_home from typing import Any, Dict, List, Optional, Tuple -from utils import base_url_host_matches, normalize_proxy_env_vars +from utils import base_url_host_matches, base_url_hostname, normalize_proxy_env_vars +from agent.secret_scope import get_secret as _get_secret + + +def _getenv(name: str, default: str = "") -> str: + """Profile-scoped replacement for os.getenv on credential reads. + + Routes through the secret scope (Workstream A): identical to os.getenv + when multiplexing is off, scope-aware (and fail-closed on an unscoped + read) when on. Mirrors the same wrapper in hermes_cli/runtime_provider.py. + """ + val = _get_secret(name, default) + return val if val is not None else default # NOTE: `import anthropic` is deliberately NOT at module top — the SDK pulls # ~220 ms of imports (anthropic.types, anthropic.lib.tools._beta_runner, etc.) @@ -368,7 +380,7 @@ def _detect_claude_code_version() -> str: try: result = _sp.run( [cmd, "--version"], - capture_output=True, text=True, timeout=5, + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=5, ) if result.returncode == 0 and result.stdout.strip(): # Output is like "2.1.74 (Claude Code)" or just "2.1.74" @@ -546,15 +558,49 @@ def _is_deepseek_anthropic_endpoint(base_url: str | None) -> bool: return "/anthropic" in normalized.rstrip("/").lower() +def _is_nous_portal_endpoint(base_url: str | None) -> bool: + """Return True for Nous Portal's Anthropic Messages route. + + Portal serves its ``anthropic/*`` catalog natively at + ``https://inference-api.nousresearch.com/v1/messages``. Portal-specific + behaviours key off this: Bearer JWT auth, verbatim catalog model ids, + and native thinking-signature replay. + + Trusted hosts only: + + 1. Prod hostname ``inference-api.nousresearch.com`` + 2. The operator-set ``NOUS_INFERENCE_BASE_URL`` hostname (staging/preview) + + Lookalikes such as ``inference-api.nousresearch.com.attacker.test`` are + rejected (hostname match, not substring). + """ + if base_url_host_matches(base_url or "", "inference-api.nousresearch.com"): + return True + try: + from hermes_cli.auth import _nous_inference_env_override + + override = _nous_inference_env_override() + except Exception: + return False + if not override: + return False + # Exact host equality (not subdomain) so the env override can't broaden + # into sibling hosts the operator did not set. + override_host = base_url_hostname(override) + return bool(override_host) and base_url_hostname(base_url or "") == override_host + + def _requires_bearer_auth(base_url: str | None) -> bool: """Return True for Anthropic-compatible providers that require Bearer auth. Some third-party /anthropic endpoints implement Anthropic's Messages API but require Authorization: Bearer instead of Anthropic's native x-api-key header. MiniMax's global and China Anthropic-compatible endpoints, Azure AI - Foundry's Anthropic-style endpoint, and Palantir Foundry's LLM proxy - follow this pattern. + Foundry's Anthropic-style endpoint, Palantir Foundry's LLM proxy, and Nous + Portal's Messages route follow this pattern. """ + if _is_nous_portal_endpoint(base_url): + return True normalized = _normalize_base_url_text(base_url) if not normalized: return False @@ -721,7 +767,11 @@ def _build_anthropic_client_with_bearer_hook( if common_betas: kwargs["default_headers"] = {"anthropic-beta": ",".join(common_betas)} - return _anthropic_sdk.Anthropic(**kwargs) + client = _anthropic_sdk.Anthropic(**kwargs) + # Same env-inference trap as build_anthropic_client: auth_token-only + # construction would otherwise also send ANTHROPIC_API_KEY as X-Api-Key. + client.api_key = None + return client def build_anthropic_client( @@ -850,7 +900,16 @@ def build_anthropic_client( if common_betas: kwargs["default_headers"] = {"anthropic-beta": ",".join(common_betas)} - return _anthropic_sdk.Anthropic(**kwargs) + client = _anthropic_sdk.Anthropic(**kwargs) + # Bearer-only construction leaves ``api_key`` unset, so the SDK fills it + # from ``ANTHROPIC_API_KEY`` (Hermes loads that into the process env from + # ``~/.hermes/.env``). The result is dual auth — + # ``X-Api-Key: sk-ant-…`` *and* ``Authorization: Bearer `` — + # on every Portal / MiniMax / OAuth Messages request. Clear the env-filled + # key whenever we intentionally authenticated via auth_token alone. + if "auth_token" in kwargs and "api_key" not in kwargs: + client.api_key = None + return client def build_anthropic_bedrock_client(region: str): @@ -914,7 +973,7 @@ def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]: "-s", "Claude Code-credentials", "-w"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=5, stdin=subprocess.DEVNULL, ) @@ -1275,7 +1334,7 @@ def _resolve_anthropic_pool_token() -> Optional[str]: # to auth.json or trigger a network refresh from a bare resolve. select() # is deliberately NOT used — it runs clear_expired=True, refresh=True, # which would violate this read-only contract. - entries = pool._available_entries(clear_expired=False, refresh=False) + entries, _pending = pool._available_entries(clear_expired=False, refresh=False) except Exception: logger.debug("Failed to read Anthropic credential_pool", exc_info=True) return None @@ -1311,7 +1370,7 @@ def resolve_anthropic_token() -> Optional[str]: creds = read_claude_code_credentials() # 1. Hermes-managed OAuth/setup token env var - token = os.getenv("ANTHROPIC_TOKEN", "").strip() + token = _getenv("ANTHROPIC_TOKEN").strip() if token: preferred = _prefer_refreshable_claude_code_token(token, creds) if preferred: @@ -1319,7 +1378,7 @@ def resolve_anthropic_token() -> Optional[str]: return token # 2. CLAUDE_CODE_OAUTH_TOKEN (used by Claude Code for setup-tokens) - cc_token = os.getenv("CLAUDE_CODE_OAUTH_TOKEN", "").strip() + cc_token = _getenv("CLAUDE_CODE_OAUTH_TOKEN").strip() if cc_token: preferred = _prefer_refreshable_claude_code_token(cc_token, creds) if preferred: @@ -1338,7 +1397,7 @@ def resolve_anthropic_token() -> Optional[str]: # 5. Regular API key, or a legacy OAuth token saved in ANTHROPIC_API_KEY. # This remains as a compatibility fallback for pre-migration Hermes configs. - api_key = os.getenv("ANTHROPIC_API_KEY", "").strip() + api_key = _getenv("ANTHROPIC_API_KEY").strip() if api_key: return api_key @@ -1381,7 +1440,7 @@ def run_oauth_setup_token() -> Optional[str]: # Check env vars that may have been set for env_var in ("CLAUDE_CODE_OAUTH_TOKEN", "ANTHROPIC_TOKEN"): - val = os.getenv(env_var, "").strip() + val = _getenv(env_var).strip() if val: return val @@ -1920,10 +1979,18 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]: return None btype = b.get("type") if btype == "text": - # Coerce empty/whitespace-only text to a non-whitespace placeholder; - # the Messages input schema rejects blank text blocks (#69512), and a - # blank block stored in history replays on every turn → permanent 400. - out: Dict[str, Any] = {"type": "text", "text": _safe_text(b.get("text", ""))} + text_val = b.get("text", "") + # Bedrock and strict Anthropic-compatible endpoints reject text + # blocks where "text" is empty or whitespace-only (#69512). Drop the + # blank block (the caller relocates any cache_control it carried and + # falls back to a non-whitespace placeholder when nothing survives) + # rather than coercing in place — a coerced "(empty)" block would be + # model-visible noise next to surviving thinking/tool_use blocks. + # Type-safe: captured blocks can carry text=None from an invalid + # upstream payload, which a bare .strip() would crash on. + if not isinstance(text_val, str) or not text_val.strip(): + return None + out: Dict[str, Any] = {"type": "text", "text": text_val} # citations is input-valid ONLY when it's a non-empty list; the SDK # emits citations=None on responses, which the input schema rejects. cits = b.get("citations") @@ -2011,9 +2078,17 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: parsed_args = {} redacted_input_by_id[_sanitize_tool_id(tc.get("id", ""))] = parsed_args replayed: List[Dict[str, Any]] = [] + _relocated_replay_cache_control = None + _dropped_blank_text = False for b in ordered_blocks: clean = _sanitize_replay_block(b) if clean is None: + if isinstance(b, dict) and b.get("type") == "text": + _dropped_blank_text = True + if isinstance(b, dict) and isinstance(b.get("cache_control"), dict): + # A dropped blank text block can still carry the cache + # breakpoint marker -- relocate it rather than losing it. + _relocated_replay_cache_control = b["cache_control"] continue if clean.get("type") == "tool_use": # Override raw (un-redacted) input with the redacted copy when @@ -2023,20 +2098,90 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: if redacted is not None: clean["input"] = redacted replayed.append(clean) + # When every text block was blank and nothing cacheable survived + # (e.g. signed thinking + a blank text block, or a SOLE blank + # cache-marked block), emit the non-whitespace placeholder so the + # replayed message stays schema-valid (#69512) and a relocated cache + # marker still has a carrier instead of being silently lost. + _has_cacheable_replay = any( + isinstance(b, dict) and b.get("type") in {"text", "tool_use"} + for b in replayed + ) + if not _has_cacheable_replay and ( + _dropped_blank_text or _relocated_replay_cache_control is not None + ): + replayed.append({"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}) if replayed: + if _relocated_replay_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + replayed, _relocated_replay_cache_control + ) _apply_assistant_cache_control_to_last_cacheable_block( replayed, m.get("cache_control") ) + # apply_anthropic_cache_control marks an assistant turn with + # non-empty text by writing cache_control INTO ``content`` (see + # _apply_cache_marker's list branch), not at the top level. This + # branch rebuilds the message from ordered_blocks and never reads + # ``content``, so that marker would be dropped -- and because + # _can_carry_marker already counted this message as a carrier, the + # breakpoint is burned rather than relocated. #56195 covered the + # complementary shape (blank content -> top-level marker); this is + # the interleaved thinking + preamble-text + tool_use shape. + _inline_cc = None + _msg_content = m.get("content") + if isinstance(_msg_content, list): + for _blk in _msg_content: + if isinstance(_blk, dict) and isinstance( + _blk.get("cache_control"), dict + ): + _inline_cc = _blk["cache_control"] + break + if _inline_cc is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + replayed, _inline_cc + ) return {"role": "assistant", "content": replayed} blocks = _extract_preserved_thinking_blocks(m) + # Cache markers dropped along with a blank block are relocated onto the + # last surviving cacheable block below (via + # _apply_assistant_cache_control_to_last_cacheable_block), rather than + # lost -- prompt_caching.py's _apply_cache_marker() sets cache_control + # directly on content[-1] for list content, so if that last part happens + # to be blank text, dropping it silently would lose the breakpoint. + _relocated_cache_control = None if content: if isinstance(content, list): converted_content = _convert_content_to_anthropic(content) if isinstance(converted_content, list): - blocks.extend(converted_content) + # Bedrock and strict Anthropic-compatible endpoints reject + # text blocks where "text" is empty or whitespace-only. The + # ordered-replay path enforces the same invariant via + # _sanitize_replay_block(). Type-safe against ANY invalid + # "text" value from an upstream payload -- None, or a + # truthy non-string like an int -- not just None: checking + # isinstance() first (rather than `blk.get("text") or ""`) + # means a non-string value is treated as blank/invalid + # instead of reaching .strip() and raising AttributeError. + for blk in converted_content: + _blk_text = blk.get("text") if isinstance(blk, dict) else None + if ( + isinstance(blk, dict) + and blk.get("type") == "text" + and (not isinstance(_blk_text, str) or not _blk_text.strip()) + ): + if isinstance(blk.get("cache_control"), dict): + _relocated_cache_control = blk["cache_control"] + continue + blocks.append(blk) else: - blocks.append({"type": "text", "text": str(content)}) + # Scalar (non-list) content: a whitespace-only string is the + # same invalid-payload case as an empty list block -- drop it + # rather than emitting a blank text block. + text_str = str(content) + if text_str.strip(): + blocks.append({"type": "text", "text": text_str}) for tc in m.get("tool_calls", []): if not tc or not isinstance(tc, dict): continue @@ -2052,9 +2197,6 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: "name": fn.get("name", ""), "input": parsed_args, }) - _apply_assistant_cache_control_to_last_cacheable_block( - blocks, m.get("cache_control") - ) # Kimi's /coding endpoint (Anthropic protocol) requires assistant # tool-call messages to carry reasoning_content when thinking is # enabled server-side. Preserve it as a thinking block so Kimi @@ -2080,19 +2222,26 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: ) if isinstance(reasoning_content, str) and not _already_has_thinking: blocks.insert(0, {"type": "thinking", "thinking": reasoning_content}) - # Anthropic rejects empty assistant content - effective = blocks or content - if not effective or effective == "": - effective = [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] - elif isinstance(effective, list): - # The all-empty guard above misses a list that still contains a - # whitespace-only text block (e.g. from a content array of blank parts, - # or compression). Those also trip "text content blocks must contain - # non-whitespace text" (#69512). Coerce text blocks in place; other - # block types (thinking/tool_use/image) are left untouched. - for blk in effective: - if isinstance(blk, dict) and blk.get("type") == "text": - blk["text"] = _safe_text(blk.get("text", "")) + # Anthropic rejects empty assistant content. IMPORTANT: fall back only + # to the placeholder, never to the raw `content` variable -- `content` + # is the UNFILTERED original message content, and can itself be exactly + # the blank/whitespace-only payload the filtering above just removed + # (a sole blank text block, or scalar whitespace with no tool_calls). + # `blocks or content` there would silently restore the invalid provider + # payload this function exists to prevent (#69512). + effective = blocks if blocks else [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] + # Applied here (after the empty-fallback resolution) rather than + # earlier against `blocks` directly, so a cache_control relocated from + # a dropped blank block that was the ONLY block still lands on the + # (empty) placeholder instead of being silently lost when blocks was + # empty at the point the marker would otherwise have been applied. + if _relocated_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + effective, _relocated_cache_control + ) + _apply_assistant_cache_control_to_last_cacheable_block( + effective, m.get("cache_control") + ) return {"role": "assistant", "content": effective} @@ -2162,13 +2311,14 @@ def _convert_user_message(content: Any) -> Dict[str, Any]: """Validate and convert a user message to anthropic format.""" if isinstance(content, list): converted_blocks = _convert_content_to_anthropic(content) - if not converted_blocks or all( - (b.get("text") or "").strip() == "" - for b in converted_blocks - if isinstance(b, dict) and b.get("type") == "text" - ): - converted_blocks = [{"type": "text", "text": "(empty message)"}] - return {"role": "user", "content": converted_blocks} + kept_blocks = _fix_blank_text_blocks_in_list( + converted_blocks, + placeholder_text="(empty message)", + msg_index=-1, + role="user", + location="_convert_user_message", + ) + return {"role": "user", "content": kept_blocks} else: if not content or (isinstance(content, str) and not content.strip()): content = "(empty message)" @@ -2326,10 +2476,22 @@ def _manage_thinking_signatures( replayed assistant tool-call messages. See hermes-agent#13848 (Kimi) and hermes-agent#16748 (DeepSeek). + Nous Portal's ``/v1/messages`` route is the exception among third-party + hosts: it proxies Claude to Anthropic/Vertex/Bedrock and validates the + same signed thinking blocks. Sticky ``session_id`` keeps a conversation + on one upstream instance so those signatures stay warm — stripping them + here would 400 the first tool-loop turn ("thinking must be passed back"). + Portal therefore takes the native Anthropic replay path below. + Mutates ``result`` in place. """ _THINKING_TYPES = frozenset(("thinking", "redacted_thinking")) - _is_third_party = _is_third_party_anthropic_endpoint(base_url) + # Portal speaks Anthropic's thinking contract end-to-end; do not treat it + # as a signature-blind proxy even though the host is not anthropic.com. + _is_third_party = ( + _is_third_party_anthropic_endpoint(base_url) + and not _is_nous_portal_endpoint(base_url) + ) last_assistant_idx = None for i in range(len(result) - 1, -1, -1): @@ -2459,9 +2621,114 @@ def _ensure_leading_user_turn(result: List[Dict[str, Any]]) -> None: Mirror the Bedrock Converse adapter, which unconditionally prepends a minimal user turn when the first message is not user (convert_messages_to_converse). + + The inserted text block must be non-whitespace: Anthropic separately + rejects any text content block whose text is empty or whitespace-only + ("text content blocks must contain non-whitespace text"), so a single + space here traded the "leading assistant turn" 400 for that one (#69512 + class). Uses the same placeholder as every other synthesized filler + block in this module for consistency. """ if result and result[0].get("role") != "user": - result.insert(0, {"role": "user", "content": [{"type": "text", "text": " "}]}) + result.insert( + 0, {"role": "user", "content": [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]} + ) + + +def _fix_blank_text_blocks_in_list( + blocks: List[Any], + *, + placeholder_text: str, + msg_index: int, + role: Any, + location: str, +) -> List[Any]: + """Drop blank/whitespace-only text blocks from ``blocks``, in place logic. + + Non-text blocks (tool_use, tool_result, image, document, thinking, …) + and the relative order of everything else are left untouched. A + cache_control marker riding on a dropped block is relocated onto the + last surviving text/tool_use block so a breakpoint is never silently + lost. If nothing survives, a single non-blank placeholder text block + takes the dropped blocks' place (carrying the relocated cache_control, + if any) so the message never has empty content. + + Returns a new list; does not mutate ``blocks``. + """ + kept: List[Any] = [] + relocated_cache_control = None + for block_index, blk in enumerate(blocks): + if ( + isinstance(blk, dict) + and blk.get("type") == "text" + and not (isinstance(blk.get("text"), str) and blk["text"].strip()) + ): + if isinstance(blk.get("cache_control"), dict): + relocated_cache_control = blk["cache_control"] + logger.warning( + "Pre-call sanitizer: dropped blank text content block " + "(message_index=%d role=%s location=%s block_index=%d " + "block_type=text)", + msg_index, + role, + location, + block_index, + ) + continue + kept.append(blk) + if not kept: + placeholder: Dict[str, Any] = {"type": "text", "text": placeholder_text} + if relocated_cache_control is not None: + placeholder["cache_control"] = relocated_cache_control + kept.append(placeholder) + elif relocated_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block(kept, relocated_cache_control) + return kept + + +def _scrub_blank_text_blocks(result: List[Dict[str, Any]]) -> None: + """Final provider-boundary guard against blank Anthropic text blocks. + + Anthropic rejects any text content block whose ``text`` is empty or + whitespace-only with HTTP 400 ("text content blocks must contain + non-whitespace text"). ``_convert_assistant_message``, + ``_convert_user_message`` and ``_ensure_leading_user_turn`` already + avoid emitting these for the paths that build them, but this pass runs + last — after every other transform in ``convert_messages_to_anthropic`` + — so a blank block from any current or future producer (including one + nested inside a ``tool_result``'s own content list) never reaches the + wire. Diagnostics are structural only: message index, role, content + location, block index/type. Never logs message text, tool arguments, + tokens, or credentials. Mutates ``result`` in place. + """ + for msg_index, msg in enumerate(result): + if not isinstance(msg, dict): + continue + role = msg.get("role") + content = msg.get("content") + if not isinstance(content, list) or not content: + continue + placeholder_text = _EMPTY_TEXT_PLACEHOLDER if role == "assistant" else "(empty message)" + new_content = _fix_blank_text_blocks_in_list( + content, + placeholder_text=placeholder_text, + msg_index=msg_index, + role=role, + location="content", + ) + for blk in new_content: + if not isinstance(blk, dict) or blk.get("type") != "tool_result": + continue + inner = blk.get("content") + if isinstance(inner, list) and inner: + blk["content"] = _fix_blank_text_blocks_in_list( + inner, + placeholder_text="(no output)", + msg_index=msg_index, + role=role, + location="tool_result", + ) + msg["content"] = new_content def convert_messages_to_anthropic( @@ -2525,6 +2792,7 @@ def convert_messages_to_anthropic( _ensure_leading_user_turn(result) _manage_thinking_signatures(result, base_url, model) _evict_old_screenshots(result) + _scrub_blank_text_blocks(result) return system, result @@ -2586,7 +2854,12 @@ def build_anthropic_kwargs( ) anthropic_tools = convert_tools_to_anthropic(tools) if tools else [] - model = normalize_model_name(model, preserve_dots=preserve_dots) + # Nous Portal routes on its own catalog ids (``anthropic/claude-opus-4.8``); + # normalizing to the bare Anthropic slug would make the model unresolvable + # there. Skipping the call preserves the prefix AND the dots, so + # ``preserve_dots`` stays irrelevant for Portal. + if not _is_nous_portal_endpoint(base_url): + model = normalize_model_name(model, preserve_dots=preserve_dots) # effective_max_tokens = output cap for this call (≠ total context window) # Use the resolver helper so non-positive values (negative ints, # fractional floats, NaN, non-numeric) fail locally with a clear error @@ -2825,6 +3098,8 @@ def create_anthropic_message( *, log_prefix: str = "", prefer_stream: bool = True, + on_stream_event=None, + on_response=None, ) -> Any: """Create an Anthropic message, aggregating via stream when available. @@ -2834,6 +3109,20 @@ def create_anthropic_message( crash on ``.content``. Prefer ``messages.stream().get_final_message()`` to match the main turn path, falling back to ``create()`` only for providers that explicitly do not support streaming, such as restricted Bedrock roles. + + ``on_stream_event``: optional callable invoked once per streamed event + (best-effort, exceptions swallowed). Lets callers report forward progress + to liveness watchdogs — e.g. the auxiliary compression path ticking its + progress hook so a slow-but-generating summary model isn't treated as + hung. Only fires on the streaming path; the ``create()`` fallback has no + events to report. + + ``on_response``: optional callable invoked once with the underlying httpx + response before the message is aggregated (best-effort, exceptions + swallowed). Response *headers* carry out-of-band provider state that the + parsed ``Message`` drops — Nous Portal's ``x-nous-credits-*`` balance family + in particular. Only fires on the streaming path, which is the one the main + turn loop takes. """ sanitize_anthropic_kwargs(api_kwargs, log_prefix=log_prefix) @@ -2844,6 +3133,26 @@ def create_anthropic_message( stream_kwargs.pop("stream", None) try: with stream_fn(**stream_kwargs) as stream: + if callable(on_response): + try: + on_response(getattr(stream, "response", None)) + except Exception: + logger.debug( + "%son_response callback failed", + log_prefix, exc_info=True, + ) + if callable(on_stream_event): + # Consume the event stream manually so each event can + # tick the caller's progress callback; get_final_message + # then returns the accumulated snapshot. + for _event in stream: + try: + on_stream_event(_event) + except Exception: + logger.debug( + "%son_stream_event callback failed", + log_prefix, exc_info=True, + ) return stream.get_final_message() except Exception as exc: if not _is_stream_unavailable_error(exc): diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 2ec78327a415..7f5794821c00 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -14,6 +14,10 @@ 6. Direct API-key providers (z.ai/GLM, Kimi/Moonshot, MiniMax, MiniMax-CN) 7. None +OpenRouter fallback cost guard: ``auxiliary.free_only: true`` restricts the +step-2 fallback to ``:free`` SKUs; ``auxiliary.openrouter_model`` overrides +the default. A one-time WARNING is logged for non-``:free`` models. + Resolution order for vision/multimodal tasks (auto mode): 1. Selected main provider, if it is one of the supported vision backends below 2. OpenRouter @@ -40,9 +44,10 @@ their OpenRouter balance but has Codex OAuth or another provider available. """ -import asyncio import contextlib import contextvars +import copy +import functools import hashlib import inspect import json @@ -51,9 +56,10 @@ import re import threading import time +import uuid from pathlib import Path # noqa: F401 — used by test mocks from types import SimpleNamespace -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse, parse_qs, urlunparse # NOTE: `from openai import OpenAI` is deliberately NOT at module top — the @@ -222,30 +228,247 @@ def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any: # part-way, compression falls back to a static "summary unavailable" marker # and the real handoff is lost (#23975). A thread-local flag lets such a # task mark its in-flight LLM call as interrupt-protected; the Codex -# Responses stream's cancellation check honors it. TIMEOUTS still fire +# Responses stream's cancellation check honors it. An explicit host cancel +# (CLI Ctrl+C or /stop) may install a cancel check that overrides protection; +# ordinary incoming-message interrupts remain protected. TIMEOUTS still fire # (a hung call must die), and all OTHER aux tasks (vision, web_extract, # title_generation, …) remain freely interruptible. _aux_interrupt_protection = threading.local() +class AuxiliaryExplicitCancellation(BaseException): + """Frozen signal that an auxiliary attempt was explicitly hard-cancelled. + + This deliberately follows ``asyncio.CancelledError`` and inherits directly + from ``BaseException``: provider retry/fallback code catches ``Exception`` + broadly and must never reinterpret an explicit host stop as a transport + failure. ``cause`` is immutable class data so downstream compression code + does not re-query a mutable host Event after the transport has unwound. + """ + + cause = "explicit_host_cancel" + + def __init__(self) -> None: + super().__init__("auxiliary request explicitly cancelled by host") + + def _aux_interrupt_protected() -> bool: return bool(getattr(_aux_interrupt_protection, "active", False)) +def _aux_interrupt_cancel_requested() -> bool: + """Return whether an explicit host cancel overrides aux protection.""" + event = getattr(_aux_interrupt_protection, "cancel_event", None) + if event is not None: + try: + return bool(event.is_set()) + except Exception: + logger.debug("aux interrupt cancel event check failed", exc_info=True) + return False + check = getattr(_aux_interrupt_protection, "cancel_check", None) + if not callable(check): + return False + try: + return bool(check()) + except Exception: + logger.debug("aux interrupt cancel check failed", exc_info=True) + return False + + @contextlib.contextmanager -def aux_interrupt_protection(active: bool = True): +def aux_interrupt_protection( + active: bool = True, + cancel_check=None, + cancel_event=None, +): """Mark the current thread's auxiliary LLM call as interrupt-protected. Used by atomic aux tasks (compression) so a mid-flight gateway interrupt doesn't abort the call and trigger a degraded fallback. Re-entrant-safe: - restores the previous value on exit. + restores the previous value on exit. ``cancel_check`` lets the host retain + an explicit hard-cancel path; ``cancel_event`` is preferred when the host + already owns an Event. Nested protection scopes inherit both values. """ prev = getattr(_aux_interrupt_protection, "active", False) + prev_cancel_check = getattr(_aux_interrupt_protection, "cancel_check", None) + prev_cancel_event = getattr(_aux_interrupt_protection, "cancel_event", None) _aux_interrupt_protection.active = active + if callable(cancel_check): + _aux_interrupt_protection.cancel_check = cancel_check + if cancel_event is not None and callable(getattr(cancel_event, "is_set", None)): + _aux_interrupt_protection.cancel_event = cancel_event try: yield finally: _aux_interrupt_protection.active = prev + _aux_interrupt_protection.cancel_check = prev_cancel_check + _aux_interrupt_protection.cancel_event = prev_cancel_event + + +def _capture_aux_cancel_check() -> Optional[Callable[[], Any]]: + """Capture the current explicit-cancel source on the owning request thread.""" + event = getattr(_aux_interrupt_protection, "cancel_event", None) + is_set = getattr(event, "is_set", None) + if callable(is_set): + return is_set + check = getattr(_aux_interrupt_protection, "cancel_check", None) + if callable(check): + # Preserve callable identity so attempt-local decision objects retain + # methods such as begin_timeout_cleanup() when captured by adapters. + return check + return None + + +def _captured_aux_cancel_requested(cancel_check: Callable[[], Any]) -> bool: + """Read a request-thread cancellation source without leaking its failures.""" + try: + return bool(cancel_check()) + except Exception: + logger.debug("captured aux cancel check failed", exc_info=True) + return False + + +class _AuxiliaryCancellationDecision: + """Atomically choose explicit cancellation or provider timeout per attempt.""" + + def __init__(self, source_cancel_check: Callable[[], Any]) -> None: + self._source_cancel_check = source_cancel_check + self._lock = threading.Lock() + self._outcome = "active" + + def __call__(self) -> bool: + with self._lock: + if self._outcome == "cancelled": + return True + if self._outcome == "timed_out": + return False + if _captured_aux_cancel_requested(self._source_cancel_check): + self._outcome = "cancelled" + return True + return False + + def begin_timeout_cleanup(self) -> bool: + """Return whether timeout won and destructive cleanup is permitted.""" + with self._lock: + if self._outcome == "active": + if _captured_aux_cancel_requested(self._source_cancel_check): + self._outcome = "cancelled" + else: + self._outcome = "timed_out" + return self._outcome == "timed_out" + + +# ── Forward-progress hook for streamed auxiliary calls ─────────────────── +# Long auxiliary calls (context compression is the prime case) are watched by +# wall-clock deadlines in their hosts (gateway session hygiene). A fixed +# deadline punishes SLOW summary models exactly as hard as HUNG ones: a +# reasoning model happily streaming a large summary is killed mid-generation. +# This thread-local hook lets the host observe liveness instead: the wire +# consumers below tick it on every streamed token/SSE event, and the host +# extends its deadline while tokens are moving (see gateway/run.py session +# hygiene + CompressionCommitFence.touch_progress). Thread-local matches the +# call topology — the aux call and its stream consumption run synchronously +# on the thread that installed the hook. +_aux_progress = threading.local() + + +def _notify_aux_progress() -> None: + """Tick the installed forward-progress hook, if any. Never raises.""" + hook = getattr(_aux_progress, "hook", None) + if hook is None: + return + try: + hook() + except Exception: + logger.debug("aux progress hook failed", exc_info=True) + + +def _aux_progress_active() -> bool: + return getattr(_aux_progress, "hook", None) is not None + + +@contextlib.contextmanager +def aux_progress_hook(hook): + """Install *hook* as the current thread's aux forward-progress callback. + + ``hook=None`` is a no-op passthrough so callers can wire it + unconditionally. Re-entrant-safe: restores the previous hook on exit. + """ + prev = getattr(_aux_progress, "hook", None) + _aux_progress.hook = hook if callable(hook) else prev + try: + yield + finally: + _aux_progress.hook = prev + + +def _run_protected_sync_provider_call( + callback: Callable[[dict[str, Any]], Any], + kwargs: dict[str, Any], +) -> Any: + """Run one protected provider callback in an attempt-isolated daemon. + + A hard cancel must release the compression-owning thread promptly, but + auxiliary clients are process-shared and cannot safely be closed or evicted + to wake one request. Only protected calls with a captured hard-cancel source + use this seam. Their provider callback (including stream aggregation) runs + in a daemon worker while the owner polls cancellation. On cancel the owner + unwinds immediately; the worker is left to finish under the provider timeout + already present in ``kwargs``. It owns no transcript or compressor commit + state and never holds the session lock. + + Ordinary auxiliary calls, and protected calls without a cancellation source, + retain the historical direct synchronous path with no extra thread. + """ + source_cancel_check = _capture_aux_cancel_check() + if not _aux_interrupt_protected() or not callable(source_cancel_check): + return callback(kwargs) + + # Freeze one linearized outcome for this isolated attempt. The host Event is + # reused and cleared on a later turn, while the Codex timeout Timer may race + # owner polling. Both paths must decide under the same attempt-local lock. + cancel_check = _AuxiliaryCancellationDecision(source_cancel_check) + + if cancel_check(): + raise AuxiliaryExplicitCancellation() + + progress_hook = getattr(_aux_progress, "hook", None) + provider_context = contextvars.copy_context() + done = threading.Event() + outcome: dict[str, Any] = {} + + def _provider_worker() -> None: + try: + with aux_progress_hook(progress_hook), aux_interrupt_protection( + cancel_check=cancel_check + ): + outcome["result"] = callback(kwargs) + except BaseException as exc: + outcome["exception"] = exc + finally: + done.set() + + threading.Thread( + target=provider_context.run, + args=(_provider_worker,), + name="hermes-protected-aux-provider", + daemon=True, + ).start() + + while True: + # Cancellation is checked before and after every completion wait so it + # wins whenever result publication and the host Event become visible in + # the same polling interval. + if _captured_aux_cancel_requested(cancel_check): + raise AuxiliaryExplicitCancellation() + if not done.wait(0.02): + continue + if _captured_aux_cancel_requested(cancel_check): + raise AuxiliaryExplicitCancellation() + exception = outcome.get("exception") + if exception is not None: + raise exception + return outcome.get("result") def _safe_isinstance(obj: Any, maybe_type: Any) -> bool: @@ -296,6 +519,10 @@ def _extract_url_query_params(url: str): "github-models": "copilot", "github-copilot-acp": "copilot-acp", "copilot-acp-agent": "copilot-acp", + "antigravity": "google-antigravity-cli", + "agy": "google-antigravity-cli", + "google-agy": "google-antigravity-cli", + "google-antigravity": "google-antigravity-cli", "tencent": "tencent-tokenhub", "tokenhub": "tencent-tokenhub", "tencent-cloud": "tencent-tokenhub", @@ -491,16 +718,17 @@ def _get_aux_model_for_provider(provider_id: str) -> str: # plus providers we intentionally keep pinned here (e.g. Anthropic predates # profiles). New providers should set default_aux_model on their profile instead. _API_KEY_PROVIDER_AUX_MODELS_FALLBACK: Dict[str, str] = { - "gemini": "gemini-3-flash-preview", + "gemini": "gemini-3.6-flash", "zai": "glm-4.5-flash", "kimi-coding": "kimi-k2-turbo-preview", "stepfun": "step-3.5-flash", "kimi-coding-cn": "kimi-k2-turbo-preview", "gmi": "google/gemini-3.1-flash-lite-preview", "anthropic": "claude-haiku-4-5-20251001", + "ai-gateway": "google/gemini-3-flash", "opencode-zen": "gemini-3-flash", "opencode-go": "glm-5", - "kilocode": "google/gemini-3-flash-preview", + "kilocode": "google/gemini-3.6-flash", "ollama-cloud": "nemotron-3-nano:30b", "tencent-tokenhub": "hy3-preview", # NB: no "deepinfra" entry — its aux model lives on the ProviderProfile @@ -631,15 +859,15 @@ def build_or_headers(or_config: dict | None = None) -> dict: Overrides ``openrouter.response_cache_ttl`` in config.yaml. *or_config* is the ``openrouter`` section from config.yaml. When *None*, - falls back to reading config from disk via ``load_config()``. + falls back to reading config from disk via ``load_config_readonly()``. """ headers = dict(_OR_HEADERS_BASE) # Resolve config from disk if not provided. if or_config is None: try: - from hermes_cli.config import load_config - or_config = load_config().get("openrouter", {}) + from hermes_cli.config import load_config_readonly + or_config = load_config_readonly().get("openrouter", {}) except Exception: or_config = {} @@ -684,6 +912,15 @@ def build_nvidia_nim_headers(base_url: str | None) -> dict: return {} +# Vercel AI Gateway app attribution headers. HTTP-Referer maps to +# referrerUrl and X-Title maps to appName in the gateway's analytics. +from hermes_cli import __version__ as _HERMES_VERSION + +_AI_GATEWAY_HEADERS = { + "HTTP-Referer": "https://hermes-agent.nousresearch.com", + "X-Title": "Hermes Agent", + "User-Agent": f"HermesAgent/{_HERMES_VERSION}", +} # Nous Portal extra_body for product attribution. # Callers should pass this as extra_body in chat.completions.create() @@ -715,8 +952,8 @@ def _nous_extra_body() -> dict: auxiliary_is_nous: bool = False # Default auxiliary models per provider -_OPENROUTER_MODEL = "google/gemini-3-flash-preview" -_NOUS_MODEL = "google/gemini-3-flash-preview" +_OPENROUTER_MODEL = "google/gemini-3.6-flash" +_NOUS_MODEL = "google/gemini-3.6-flash" _NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1" _ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com" _AUTH_JSON_PATH = get_hermes_home() / "auth.json" @@ -900,6 +1137,29 @@ def _nous_min_key_ttl_seconds() -> int: return 1800 +def _scoped_key_env(name: str) -> str: + """Read a provider API key env var through the profile secret scope. + + Auxiliary-client resolution runs both inside agent turns (secret scope + installed — its verdict is authoritative under multiplex, so a scoped + miss must NOT borrow another profile's process-env key) and on unscoped + startup/CLI probe paths, which keep the legacy ``os.environ`` read via + the ``UnscopedSecretError`` fallback (Slack pattern, #59739). + """ + if not name: + return "" + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return (os.getenv(name) or "").strip() + + # ── Codex Responses → chat.completions adapter ───────────────────────────── # All auxiliary consumers call client.chat.completions.create(**kwargs) and # read response.choices[0].message.content. This adapter translates those @@ -1059,16 +1319,29 @@ def create(self, **kwargs) -> Any: # key in extra_body (not top-level) and GitHub/Copilot Responses opts # out of cache-key routing entirely — for those hosts, skip it here. try: - from agent.transports.codex import _content_cache_key + from agent.transports.codex import ( + _content_cache_key, + _default_prompt_cache_retention_for_request, + ) from utils import base_url_host_matches _host_src = str(getattr(self._client, "base_url", "") or "") _is_xai = base_url_host_matches(_host_src, "x.ai") or base_url_host_matches(_host_src, "api.x.ai") - _is_github = base_url_host_matches(_host_src, "githubcopilot.com") + _is_github = ( + base_url_host_matches(_host_src, "githubcopilot.com") + or base_url_host_matches(_host_src, "models.github.ai") + ) if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs: _cache_key = _content_cache_key(instructions, resp_kwargs.get("tools")) if _cache_key: resp_kwargs["prompt_cache_key"] = _cache_key + if "prompt_cache_retention" not in resp_kwargs: + _cache_retention = _default_prompt_cache_retention_for_request( + model, + _host_src, + ) + if _cache_retention: + resp_kwargs["prompt_cache_retention"] = _cache_retention except Exception: logger.debug( "Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True @@ -1082,12 +1355,53 @@ def create(self, **kwargs) -> Any: deadline = time.monotonic() + float(total_timeout) if total_timeout else None timed_out = threading.Event() timeout_timer: Optional[threading.Timer] = None + # A protected provider call may outlive its owning compression attempt: + # the owner returns promptly on hard cancellation while this adapter is + # still blocked in the SDK stream on its isolated worker. Timer threads + # do not inherit this worker's thread-local protection state, so freeze + # the hard-cancel source here, before creating the timer. + protected_cancel_check = ( + _capture_aux_cancel_check() if _aux_interrupt_protected() else None + ) + attempt_stream_lock = threading.Lock() + attempt_stream: List[Any] = [] def _timeout_message() -> str: return f"Codex auxiliary Responses stream exceeded {float(total_timeout):.1f}s total timeout" def _close_client_on_timeout() -> None: + begin_timeout_cleanup = getattr( + protected_cancel_check, "begin_timeout_cleanup", None + ) + if callable(begin_timeout_cleanup): + timeout_won = bool(begin_timeout_cleanup()) + else: + timeout_won = not ( + callable(protected_cancel_check) + and _captured_aux_cancel_requested(protected_cancel_check) + ) + # Publish transport timeout only after the attempt-local decision is + # fixed, so owner polling cannot observe completion in between. timed_out.set() + if not timeout_won: + # The request owner already hard-cancelled this attempt. The + # OpenAI client is process-shared, so closing/evicting it here + # would disrupt unrelated sessions. Wake only this attempt's + # event stream when responses.create() returned one in time; + # otherwise rely on the bounded SDK/provider timeout. + with attempt_stream_lock: + stream = attempt_stream[0] if attempt_stream else None + close_stream = getattr(stream, "close", None) + if callable(close_stream): + try: + close_stream() + except Exception: + logger.debug( + "Codex auxiliary: cancelled attempt stream close " + "during timeout failed", + exc_info=True, + ) + return close = getattr(self._client, "close", None) if callable(close): try: @@ -1114,11 +1428,14 @@ def _check_cancelled() -> None: from tools.interrupt import is_interrupted # Honor interrupt protection for atomic aux tasks (compression): # a mid-flight gateway interrupt must NOT abort the summary call - # and trigger a degraded fallback marker (#23975). Timeouts above - # still fire; other aux tasks remain interruptible. + # and trigger a degraded fallback marker (#23975). Explicit host + # cancellation has its own frozen exception; timeouts above still + # fire and other aux tasks remain interruptible. + if _aux_interrupt_cancel_requested(): + raise AuxiliaryExplicitCancellation() if is_interrupted() and not _aux_interrupt_protected(): raise InterruptedError("Codex auxiliary Responses stream interrupted") - except InterruptedError: + except (InterruptedError, AuxiliaryExplicitCancellation): raise except Exception: # Interrupt state is a best-effort UX hook; never make it a @@ -1150,15 +1467,46 @@ def _check_cancelled() -> None: def _on_each_event(_event: Any) -> None: # Re-check timeout/cancellation per event, matching the # cadence the old in-line ``_check_cancelled()`` used. + # Each SSE event is also forward progress for hosts watching + # a progress hook (gateway session hygiene): a reasoning + # model streaming a long summary must not look hung. + _notify_aux_progress() _check_cancelled() event_stream = self._client.responses.create(**stream_kwargs) + with attempt_stream_lock: + attempt_stream.append(event_stream) + # The timer can fire while responses.create() is blocked. If the + # cancelled attempt had no stream to close at that instant, close it + # now that it is safely attempt-owned; never touch the shared client. + if ( + timed_out.is_set() + and callable(protected_cancel_check) + and _captured_aux_cancel_requested(protected_cancel_check) + ): + close_fn = getattr(event_stream, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + logger.debug( + "Codex auxiliary: late cancelled attempt stream close failed", + exc_info=True, + ) try: - final = _consume_codex_event_stream( - event_stream, - model=resp_kwargs.get("model"), - on_event=_on_each_event, - ) + # Some Codex-compatible hosts accept ``stream=True`` but return + # a completed Responses object instead of an SSE iterator. Do + # not hand that object to the event consumer: typed Responses + # (and compatibility shims such as SimpleNamespace) are not + # event streams and may not be iterable at all. + if hasattr(event_stream, "output"): + final = event_stream + else: + final = _consume_codex_event_stream( + event_stream, + model=str(resp_kwargs.get("model") or model), + on_event=_on_each_event, + ) finally: close_fn = getattr(event_stream, "close", None) if callable(close_fn): @@ -1166,6 +1514,8 @@ def _on_each_event(_event: Any) -> None: close_fn() except Exception: pass + with attempt_stream_lock: + attempt_stream.clear() if final is None: raise RuntimeError("Codex auxiliary Responses stream did not return a final response") @@ -1302,10 +1652,32 @@ def __init__(self, sync_wrapper: "CodexAuxiliaryClient"): class _AnthropicCompletionsAdapter: """OpenAI-client-compatible adapter for Anthropic Messages API.""" - def __init__(self, real_client: Any, model: str, is_oauth: bool = False): + def __init__( + self, + real_client: Any, + model: str, + is_oauth: bool = False, + base_url: str | None = None, + ): self._client = real_client self._model = model self._is_oauth = is_oauth + # Prefer the caller-supplied URL (AnthropicAuxiliaryClient keeps the + # pre-strip Portal ``.../v1`` form). Only fall back to the SDK + # client's host for Nous Portal — a blanket fallback would flip + # MiniMax/Zhipu/etc. aux adapters from "unknown host = native + # Anthropic" to third-party (stripping thinking signatures). + self._base_url = base_url or None + if not self._base_url: + candidate = str(getattr(real_client, "base_url", "") or "") or None + if candidate: + try: + from agent.anthropic_adapter import _is_nous_portal_endpoint + + if _is_nous_portal_endpoint(candidate): + self._base_url = candidate + except Exception: + pass def create(self, **kwargs) -> Any: from agent.anthropic_adapter import build_anthropic_kwargs, create_anthropic_message @@ -1357,6 +1729,11 @@ def create(self, **kwargs) -> Any: reasoning_config=_reasoning_cfg, tool_choice=normalized_tool_choice, is_oauth=self._is_oauth, + # Portal routes on ``anthropic/`` catalog ids and replays + # signed thinking like native Anthropic; both carve-outs key off + # base_url. Omitting it normalizes the id to a bare Anthropic + # slug and the Portal Messages route cannot resolve it. + base_url=self._base_url, ) # Opus 4.7+ rejects any non-default temperature/top_p/top_k; only set # temperature for models that still accept it. build_anthropic_kwargs @@ -1391,7 +1768,18 @@ def create(self, **kwargs) -> Any: existing = {} anthropic_kwargs["extra_body"] = {**existing, **passthrough} - response = create_anthropic_message(self._client, anthropic_kwargs) + response = create_anthropic_message( + self._client, + anthropic_kwargs, + # Tick the aux forward-progress hook per streamed event so hosts + # watching liveness (gateway session hygiene) don't kill a + # slow-but-generating summary model. No-op when no hook is + # installed (None keeps the fast get_final_message path). + on_stream_event=( + (lambda _event: _notify_aux_progress()) + if _aux_progress_active() else None + ), + ) _transport = get_transport("anthropic_messages") _nr = _transport.normalize_response( response, strip_tool_prefix=self._is_oauth @@ -1439,7 +1827,9 @@ class AnthropicAuxiliaryClient: def __init__(self, real_client: Any, model: str, api_key: str, base_url: str, is_oauth: bool = False): self._real_client = real_client - adapter = _AnthropicCompletionsAdapter(real_client, model, is_oauth=is_oauth) + adapter = _AnthropicCompletionsAdapter( + real_client, model, is_oauth=is_oauth, base_url=base_url, + ) self.chat = _AnthropicChatShim(adapter) self.api_key = api_key self.base_url = base_url @@ -1704,7 +2094,7 @@ def _read_nous_auth() -> Optional[dict]: try: if not _AUTH_JSON_PATH.is_file(): return None - data = json.loads(_AUTH_JSON_PATH.read_text()) + data = json.loads(_AUTH_JSON_PATH.read_text(encoding="utf-8")) if data.get("active_provider") != "nous": return None provider = data.get("providers", {}).get("nous", {}) @@ -2038,8 +2428,63 @@ def _resolve_api_key_provider() -> Tuple[Optional[OpenAI], Optional[str]]: # ── Provider resolution helpers ───────────────────────────────────────────── +_paid_lane_warned: set = set() + + +def _is_free_model(model: Optional[str]) -> bool: + """True when ``model`` is an OpenRouter free SKU (``:free`` suffix).""" + return bool(model) and str(model).strip().endswith(":free") + + +def _aux_openrouter_settings() -> Tuple[bool, str]: + """Read free_only and openrouter_model from config in one pass. + + Returns (free_only, model) — defaults (False, _OPENROUTER_MODEL) on any + config-read failure. + """ + try: + from hermes_cli.config import cfg_get, load_config_readonly + + cfg = load_config_readonly() + free_only = bool(cfg_get(cfg, "auxiliary", "free_only", default=False)) + val = cfg_get(cfg, "auxiliary", "openrouter_model") + model = val.strip() if isinstance(val, str) and val.strip() else _OPENROUTER_MODEL + return free_only, model + except Exception: + return False, _OPENROUTER_MODEL + + +def _warn_paid_lane_once(model: str) -> None: + """Log a WARNING the first time a non-:free OpenRouter model is engaged.""" + if model in _paid_lane_warned: + return + _paid_lane_warned.add(model) + logger.warning( + "Auxiliary client: PAID lane engaged for auxiliary task — OpenRouter " + "fallback model %r is not a :free SKU and may incur real spend. Set " + "auxiliary.free_only: true to restrict auxiliary fallbacks to free " + "models, or auxiliary.openrouter_model to a :free model.", + model, + ) + def _try_openrouter(explicit_api_key: str = None, model: str = None) -> Tuple[Optional[OpenAI], Optional[str]]: + free_only, cfg_model = _aux_openrouter_settings() + or_model = model or cfg_model + if free_only and not _is_free_model(or_model): + logger.warning( + "Auxiliary client: auxiliary.free_only is enabled but the " + "OpenRouter fallback model %r is not a :free SKU — skipping the " + "OpenRouter fallback. Set auxiliary.openrouter_model to a :free " + "model (e.g. nvidia/nemotron-3-ultra-550b-a55b:free) or disable " + "auxiliary.free_only.", + or_model, + ) + _mark_provider_unhealthy("openrouter", ttl=60) + return None, None + if not _is_free_model(or_model): + _warn_paid_lane_once(or_model) + pool_present, entry = _select_pool_entry("openrouter") if pool_present: or_key = explicit_api_key or _pool_runtime_api_key(entry) @@ -2047,18 +2492,18 @@ def _try_openrouter(explicit_api_key: str = None, model: str = None) -> Tuple[Op base_url = _pool_runtime_base_url(entry, OPENROUTER_BASE_URL) or OPENROUTER_BASE_URL logger.debug("Auxiliary client: OpenRouter via pool") return _create_openai_client(api_key=or_key, base_url=base_url, - default_headers=build_or_headers()), model or _OPENROUTER_MODEL + default_headers=build_or_headers()), or_model # Pool exists but is exhausted (no usable runtime key) — fall through to # the OPENROUTER_API_KEY env-var path rather than failing outright. logger.debug("Auxiliary client: OpenRouter pool exhausted, trying OPENROUTER_API_KEY") - or_key = explicit_api_key or os.getenv("OPENROUTER_API_KEY") + or_key = explicit_api_key or _scoped_key_env("OPENROUTER_API_KEY") if not or_key: _mark_provider_unhealthy("openrouter", ttl=60) return None, None logger.debug("Auxiliary client: OpenRouter") return _create_openai_client(api_key=or_key, base_url=OPENROUTER_BASE_URL, - default_headers=build_or_headers()), model or _OPENROUTER_MODEL + default_headers=build_or_headers()), or_model def _describe_openrouter_unavailable() -> str: @@ -2069,7 +2514,7 @@ def _describe_openrouter_unavailable() -> str: return "OpenRouter credential pool has no usable entries (credentials may be exhausted)" if not _pool_runtime_api_key(entry): return "OpenRouter credential pool entry is missing a runtime API key" - if not str(os.getenv("OPENROUTER_API_KEY") or "").strip(): + if not _scoped_key_env("OPENROUTER_API_KEY"): return "OPENROUTER_API_KEY not set" return "no usable OpenRouter credentials found" @@ -2215,8 +2660,8 @@ def _read_main_model() -> str: if isinstance(override, str) and override.strip(): return override.strip() try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() model_cfg = cfg.get("model", {}) if isinstance(model_cfg, str) and model_cfg.strip(): return model_cfg.strip() @@ -2242,8 +2687,8 @@ def _read_main_provider() -> str: if isinstance(override, str) and override.strip(): return override.strip().lower() try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() model_cfg = cfg.get("model", {}) if isinstance(model_cfg, dict): provider = model_cfg.get("provider", "") @@ -2392,6 +2837,171 @@ def _read_main_api_key_if_same_host(aux_base_url: str) -> str: _RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( contextvars.ContextVar("auxiliary_runtime_main", default=None) ) + +_RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = ( + contextvars.ContextVar("auxiliary_relay_call", default=None) +) + + +def _relay_auxiliary_call(callback): + """Give every physical retry in one auxiliary call a shared Relay identity.""" + + @functools.wraps(callback) + def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return callback(*args, **kwargs) + except BaseException: + _fail_relay_auxiliary_call() + raise + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _relay_auxiliary_call_async(callback): + """Async counterpart to :func:`_relay_auxiliary_call`.""" + + @functools.wraps(callback) + async def wrapped(*args, **kwargs): + task = args[0] if args else kwargs.get("task") + token = _RELAY_AUX_CALL_CONTEXT.set({ + "task": str(task or "unknown"), + "request_id": f"aux-{uuid.uuid4().hex}", + "attempt_count": 0, + "provider": "", + "model": "", + "api_mode": "chat_completions", + }) + try: + return await callback(*args, **kwargs) + except BaseException: + _fail_relay_auxiliary_call() + raise + finally: + _RELAY_AUX_CALL_CONTEXT.reset(token) + + return wrapped + + +def _set_relay_auxiliary_route( + provider: str | None, + model: str | None, + api_mode: str | None, +) -> None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + context["provider"] = str(provider or "auxiliary") + context["model"] = str(model or "unknown") + context["api_mode"] = str(api_mode or "chat_completions") + + +def _relay_auxiliary_metadata( + *, + provider: str | None = None, + api_mode: str | None = None, +) -> tuple[str, str, dict[str, Any]] | None: + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return None + attempt_count = int(context.get("attempt_count") or 0) + context["attempt_count"] = attempt_count + 1 + provider_name = str(provider or context.get("provider") or "auxiliary") + model_name = str(context.get("model") or "unknown") + return provider_name, model_name, { + "api_mode": str(api_mode or context.get("api_mode") or "chat_completions"), + "api_request_id": str(context["request_id"]), + "call_role": f"auxiliary:{context['task']}", + "retry_count": attempt_count, + "auxiliary_task": str(context["task"]), + } + + +def _relay_sync_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + # Protected compression calls isolate only the provider callback and stream + # aggregation. The owning thread remains free to unwind its lease/DB + # transaction on hard cancel without touching the process-shared client. + if route is None: + return _run_protected_sync_provider_call(callback, kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.execute_current( + kwargs, + lambda request: _run_protected_sync_provider_call(callback, request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + defer_logical_completion=True, + ) + + +async def _relay_async_completion( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, + create: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + callback = create or (lambda request: client.chat.completions.create(**request)) + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return await callback(kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return await relay_llm.execute_current_async( + kwargs, + callback, + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + metadata=metadata, + defer_logical_completion=True, + ) + + +def _relay_sync_stream( + client: Any, + kwargs: dict[str, Any], + *, + provider: str | None = None, + api_mode: str | None = None, +) -> Any: + route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + if route is None: + return client.chat.completions.create(**kwargs) + provider_name, fallback_model, metadata = route + from agent import relay_llm + + return relay_llm.stream_current( + kwargs, + lambda request: client.chat.completions.create(**request), + name=provider_name, + model_name=str(kwargs.get("model") or fallback_model), + finalizer=dict, + metadata=metadata, + completed_response_predicate=lambda value: hasattr(value, "choices"), + ) _RUNTIME_MAIN_COMPAT_SNAPSHOT: Tuple[Any, ...] = ("", "", "", "", "", "") _RUNTIME_MAIN_COMPAT_LOCK = threading.Lock() @@ -2536,7 +3146,7 @@ def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[st if not isinstance(runtime, dict): openai_base = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/") - openai_key = os.getenv("OPENAI_API_KEY", "").strip() + openai_key = _scoped_key_env("OPENAI_API_KEY") if not openai_base: return None, None, None runtime = { @@ -2694,7 +3304,13 @@ def _build_xai_oauth_aux_client(model: str) -> Tuple[Optional[Any], Optional[str return None, None api_key, base_url = resolved logger.debug("Auxiliary client: xAI OAuth (%s via Responses API)", model) - real_client = _create_openai_client(api_key=api_key, base_url=base_url) + from tools.xai_http import hermes_xai_default_headers + + real_client = _create_openai_client( + api_key=api_key, + base_url=base_url, + default_headers=hermes_xai_default_headers(), + ) return CodexAuxiliaryClient(real_client, model), model @@ -2771,12 +3387,12 @@ def _try_azure_foundry( try: from hermes_cli.runtime_provider import _resolve_azure_foundry_runtime from hermes_cli.auth import AuthError - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly except ImportError: return None, None try: - cfg = load_config() + cfg = load_config_readonly() model_cfg = cfg.get("model") if isinstance(cfg, dict) else {} if not isinstance(model_cfg, dict): model_cfg = {} @@ -2890,8 +3506,8 @@ def _try_anthropic(explicit_api_key: str = None) -> Tuple[Optional[Any], Optiona # see issue #52608. base_url = _pool_runtime_base_url(entry, _ANTHROPIC_DEFAULT_BASE_URL) if pool_present else _ANTHROPIC_DEFAULT_BASE_URL try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() model_cfg = cfg.get("model") if isinstance(model_cfg, dict): cfg_provider = str(model_cfg.get("provider") or "").strip().lower() @@ -3033,12 +3649,9 @@ def _normalize_chain_label(provider: str) -> str: def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None: - """Temporarily hide an unavailable ``provider`` from chain iteration. - - Called after confirmed payment exhaustion, stale credentials, or a - deployment-level capability failure. The in-process TTL prevents an - immediate fallback loop back into the same broken route while still - allowing automatic recovery. + """Mark ``provider`` as recently-402'd, hidden from chain iteration + until the TTL expires. Called from the payment-fallback branches in + ``call_llm`` and ``acall_llm`` after a confirmed payment error. """ label = _normalize_chain_label(provider) if not label: @@ -3046,7 +3659,7 @@ def _mark_provider_unhealthy(provider: str, ttl: Optional[float] = None) -> None expires_at = time.time() + (ttl if ttl is not None else _AUX_UNHEALTHY_TTL_SECONDS) _aux_unhealthy_until[label] = expires_at logger.warning( - "Auxiliary: marking %s unhealthy for %ds. " + "Auxiliary: marking %s unhealthy for %ds (payment / credit error). " "Subsequent auxiliary calls will skip it until %s.", label, int(ttl if ttl is not None else _AUX_UNHEALTHY_TTL_SECONDS), @@ -3058,15 +3671,14 @@ def _is_provider_unhealthy(label: str) -> bool: """True iff ``label`` is in the unhealthy cache and the TTL hasn't expired. Lazily evicts expired entries so the cache stays small. """ - normalized = _normalize_chain_label(label) - if not normalized: + if not label: return False - expires_at = _aux_unhealthy_until.get(normalized) + expires_at = _aux_unhealthy_until.get(label) if expires_at is None: return False if time.time() >= expires_at: - _aux_unhealthy_until.pop(normalized, None) - _aux_unhealthy_logged_at.pop(normalized, None) + _aux_unhealthy_until.pop(label, None) + _aux_unhealthy_logged_at.pop(label, None) return False return True @@ -3076,15 +3688,14 @@ def _log_skip_unhealthy(label: str, task: Optional[str] = None) -> None: provider. Avoids spamming the log on bursty sessions while still giving the user a trail. """ - normalized = _normalize_chain_label(label) now = time.time() - last = _aux_unhealthy_logged_at.get(normalized, 0.0) + last = _aux_unhealthy_logged_at.get(label, 0.0) if now - last >= 60: - _aux_unhealthy_logged_at[normalized] = now - expires_at = _aux_unhealthy_until.get(normalized, now) + _aux_unhealthy_logged_at[label] = now + expires_at = _aux_unhealthy_until.get(label, now) logger.info( - "Auxiliary %s: skipping %s (recently unavailable, retry in %ds)", - task or "call", normalized, max(0, int(expires_at - now)), + "Auxiliary %s: skipping %s (recently returned payment error, retry in %ds)", + task or "call", label, max(0, int(expires_at - now)), ) @@ -3245,27 +3856,6 @@ def _is_connection_error(exc: Exception) -> bool: return False -def _http_status_code(exc: Exception) -> Optional[int]: - """Extract an HTTP status from SDK exceptions and plain adapter errors. - - Native OpenAI-compatible exceptions expose ``status_code`` (sometimes on - ``response``), but subprocess/native adapters commonly preserve it only in - text such as ``Gemini HTTP 503`` or ``Error code: 404``. Keeping this in one - helper prevents retry and fallback gates from silently disagreeing. - """ - status = getattr(exc, "status_code", None) or getattr( - getattr(exc, "response", None), "status_code", None - ) - if isinstance(status, int): - return status - match = re.search( - r"\b(?:http|status|error\s+code)\s*[:=]?\s*([1-5]\d{2})\b", - str(exc), - re.IGNORECASE, - ) - return int(match.group(1)) if match else None - - def _is_transient_transport_error(exc: Exception) -> bool: """Return True for a one-off transport blip worth retrying ON the same provider before any provider/model fallback. @@ -3279,7 +3869,9 @@ def _is_transient_transport_error(exc: Exception) -> bool: """ if _is_connection_error(exc): return True - status = _http_status_code(exc) + status = getattr(exc, "status_code", None) or getattr( + getattr(exc, "response", None), "status_code", None + ) return isinstance(status, int) and (status == 408 or 500 <= status < 600) @@ -3466,32 +4058,6 @@ def _is_model_incompatible_error(exc: Exception) -> bool: )) -def _is_provider_deployment_unavailable_error(exc: Exception) -> bool: - """Detect a configured provider route whose backing deployment vanished. - - NVIDIA NIM can keep accepting a model slug while returning HTTP 404 because - the account-scoped function deployment behind it was retired. The client - resolves and authenticates, so this is a hard route-capability failure and - auxiliary tasks must continue through their configured fallback chain. - - Any 5xx is a temporary deployment outage after bounded same-provider - retries. Keep 404 deliberately narrow: a generic 404 may be a caller or - configuration mistake and must not silently bypass an explicitly selected - provider. - """ - status = _http_status_code(exc) - if isinstance(status, int) and 500 <= status < 600: - return True - if status != 404: - return False - err_lower = str(exc).lower() - return ( - "function id" in err_lower - and "specified function" in err_lower - and "not found" in err_lower - ) - - def _is_invalid_aux_response_error(exc: Exception) -> bool: """Detect provider responses that authenticated but cannot serve aux shape. @@ -3590,38 +4156,16 @@ def _pool_error_context(exc: Exception) -> Dict[str, Any]: return payload -def _route_has_explicit_endpoint_or_key( - base_url: Optional[str], - api_key: Optional[str], -) -> bool: - """Return whether an auxiliary route overrides provider-owned routing.""" - return bool(str(base_url or "").strip() or str(api_key or "").strip()) - - def _recoverable_pool_provider( resolved_provider: str, client: Any, main_runtime: Optional[Dict[str, Any]] = None, - *, - route_is_explicit: bool = False, ) -> Optional[str]: """Infer which provider pool can recover the current auxiliary client.""" - route_label = str(resolved_provider or "").strip().lower() - if route_label == "custom" or route_label.startswith("custom:"): - # An explicit custom route owns its own endpoint/key. Even when its - # base URL happens to share a hostname with a built-in provider, it - # must never rotate or exhaust that unrelated provider's pool. - return None normalized = _normalize_aux_provider(resolved_provider) - base = str(getattr(client, "base_url", "") or "") - normalized_base = _normalized_provider_route(base) - - # Concrete provider clients can legitimately select provider-owned dynamic - # routes (for example Z.AI global/China/coding endpoints). Preserve their - # pool association unless the task supplied an explicit route/key override. if normalized not in {"", "auto", "custom"}: - return None if route_is_explicit else normalized - + return normalized + base = str(getattr(client, "base_url", "") or "") if base_url_host_matches(base, "chatgpt.com"): return "openai-codex" if base_url_host_matches(base, "openrouter.ai"): @@ -3636,69 +4180,25 @@ def _recoverable_pool_provider( return "kimi-coding" if base_url_host_matches(base, "api.x.ai"): return "xai-oauth" - # Match the selected client's full base route against registered providers - # even when ``resolved_provider`` is ``auto``. Host-only matching is unsafe: - # opencode-zen and opencode-go deliberately share opencode.ai but use - # different path roots. Prefer the longest normalized route so nested - # provider paths resolve to their exact owner. - preferred_provider = "" + # For api_key providers not in the hardcoded list (e.g. opencode-go), match + # the client base URL against all registered api_key providers so that + # credential-pool rotation works for any provider the user configured. if main_runtime: rt = _normalize_main_runtime(main_runtime) - preferred_provider = str(rt.get("provider") or "") - try: - from hermes_cli.auth import PROVIDER_REGISTRY - - provider_ids = list(PROVIDER_REGISTRY) - if preferred_provider in PROVIDER_REGISTRY: - provider_ids.remove(preferred_provider) - provider_ids.insert(0, preferred_provider) - best_match: tuple[int, str] | None = None - for provider_id in provider_ids: - pconfig = PROVIDER_REGISTRY.get(provider_id) - candidates = [ - str(getattr(pconfig, "inference_base_url", "") or ""), - ] - base_url_env_var = str( - getattr(pconfig, "base_url_env_var", "") or "" - ).strip() - if base_url_env_var: - candidates.append(str(os.environ.get(base_url_env_var) or "")) - for candidate in candidates: - normalized_candidate = _normalized_provider_route(candidate) - if not normalized_candidate or not normalized_base: - continue - if ( - normalized_base == normalized_candidate - or normalized_base.startswith(normalized_candidate + "/") - ): - score = len(normalized_candidate) - if best_match is None or score > best_match[0]: - best_match = (score, provider_id) - if best_match is not None: - return best_match[1] - except Exception: - pass + rt_provider = rt.get("provider", "") + if rt_provider and rt_provider not in {"", "auto", "custom"}: + try: + from hermes_cli.auth import PROVIDER_REGISTRY + pconfig = PROVIDER_REGISTRY.get(rt_provider) + if pconfig and getattr(pconfig, "auth_type", None) == "api_key": + rt_base = str(getattr(pconfig, "inference_base_url", "") or "").rstrip("/") + if rt_base and base_url_host_matches(base, base_url_hostname(rt_base)): + return rt_provider + except Exception: + pass return None -def _normalized_provider_route(value: str) -> str: - """Normalize an HTTP provider base URL without discarding its path.""" - try: - parsed = urlparse(str(value or "").strip()) - except Exception: - return "" - if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: - return "" - return urlunparse(( - parsed.scheme.lower(), - parsed.netloc.lower(), - parsed.path.rstrip("/"), - "", - "", - "", - )) - - def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str = "") -> bool: """Try same-provider credential-pool recovery for auxiliary calls. @@ -3811,7 +4311,13 @@ def _retry_same_provider_sync( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - retry_client.chat.completions.create(**retry_kwargs), task, + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -3876,7 +4382,13 @@ async def _retry_same_provider_async( if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( - await retry_client.chat.completions.create(**retry_kwargs), task, + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), + task, ) @@ -3996,23 +4508,13 @@ def _auth_refresh_provider_for_route( return normalized -def _fallback_entry_timeout(task: Optional[str], fb_label: str) -> Optional[float]: - """Resolve a per-entry ``timeout`` for a configured fallback candidate. +def _fallback_chain_entry(task: Optional[str], fb_label: str) -> Optional[Dict[str, Any]]: + """Resolve the configured ``fallback_chain`` entry a label points at. - A fallback candidate previously inherited the exact timeout the primary - provider was called with. When that deadline was tuned for the primary - (or the primary simply consumed its whole budget before failing over), - the fallback aborted on the same clock even when independently healthy — - a 163k-token compression that needs ~90s on the fallback died at the - primary's 30s deadline every turn (#62452). - - Entries in ``auxiliary..fallback_chain`` may declare their own - ``timeout`` (seconds). This helper reads it by parsing the entry index - out of the label minted by :func:`_try_configured_fallback_chain` - (``fallback_chain[]()`` — our own stable format). Returns - ``None`` when the label is not a configured-chain candidate, the entry - has no ``timeout``, or the value is invalid — callers then keep the - task-level timeout, preserving existing behavior. + Labels minted by :func:`_try_configured_fallback_chain` carry the entry + index in our own stable format (``fallback_chain[]()``). + Returns ``None`` when the label is not a configured-chain candidate or + the index no longer resolves to a dict entry. """ if not task or not fb_label: return None @@ -4022,205 +4524,127 @@ def _fallback_entry_timeout(task: Optional[str], fb_label: str) -> Optional[floa try: chain = _get_auxiliary_task_config(task).get("fallback_chain") entry = chain[int(m.group(1))] if isinstance(chain, list) else None - raw = entry.get("timeout") if isinstance(entry, dict) else None except Exception: return None + return entry if isinstance(entry, dict) else None + + +def _fallback_entry_timeout(task: Optional[str], fb_label: str) -> Optional[float]: + """Resolve a per-entry ``timeout`` for a configured fallback candidate. + + A fallback candidate previously inherited the exact timeout the primary + provider was called with. When that deadline was tuned for the primary + (or the primary simply consumed its whole budget before failing over), + the fallback aborted on the same clock even when independently healthy — + a 163k-token compression that needs ~90s on the fallback died at the + primary's 30s deadline every turn (#62452). + + Entries in ``auxiliary..fallback_chain`` may declare their own + ``timeout`` (seconds). Returns ``None`` when the label is not a + configured-chain candidate, the entry has no ``timeout``, or the value + is invalid — callers then keep the task-level timeout, preserving + existing behavior. + """ + entry = _fallback_chain_entry(task, fb_label) + raw = entry.get("timeout") if entry else None if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw > 0: return float(raw) return None -def _fallback_label_provider(fb_label: str) -> str: - """Extract the declared provider from a configured fallback label.""" - match = re.search(r"\(([^()]+)\)$", str(fb_label or "")) - return match.group(1) if match else str(fb_label or "") +def _fallback_provider_from_label(label: str) -> str: + """Recover the provider identifier from a fallback display label.""" + match = re.match(r"(?:fallback_chain\[\d+\]|main-agent)\(([^)]+)\)$", label or "") + return match.group(1).strip() if match else str(label or "").strip() -def _fallback_entry_for_health( - task: Optional[str], - fb_label: str, -) -> Optional[Dict[str, Any]]: - """Return the configured entry represented by a stable fallback label.""" - label = str(fb_label or "").strip() - task_match = re.match(r"fallback_chain\[(\d+)\]", label) - if task_match and task: - try: - chain = _get_auxiliary_task_config(task).get("fallback_chain") - entry = chain[int(task_match.group(1))] if isinstance(chain, list) else None - return entry if isinstance(entry, dict) else None - except Exception: - return None - - main_match = re.match(r"fallback_providers\[(\d+)\]", label) - if main_match: - try: - from hermes_cli.config import load_config - from hermes_cli.fallback_config import get_fallback_chain +class _FallbackDestination(NamedTuple): + provider: str + base_url: str + api_mode: Optional[str] + model: Optional[str] - chain = get_fallback_chain(load_config()) - entry = chain[int(main_match.group(1))] if isinstance(chain, list) else None - return entry if isinstance(entry, dict) else None - except Exception: - return None - return None +def _complete_fallback_destination( + provider: str, + base_url: str, + api_mode: Optional[str], + model: Optional[str], +) -> _FallbackDestination: + if not api_mode: + if _endpoint_speaks_anthropic_messages(base_url): + api_mode = "anthropic_messages" + else: + try: + from hermes_cli.runtime_provider import resolve_runtime_provider -def _fallback_entry_api_key(entry: Dict[str, Any]) -> Optional[str]: - """Resolve inline or env-backed API key from a fallback-chain entry.""" - explicit = str(entry.get("api_key") or "").strip() - if explicit: - return explicit - key_env = str(entry.get("key_env") or entry.get("api_key_env") or "").strip() - if key_env: - return os.getenv(key_env, "").strip() or None - return None + runtime = resolve_runtime_provider( + requested=provider, + explicit_base_url=base_url or None, + target_model=model or "", + ) + api_mode = str(runtime.get("api_mode") or "").strip() or None + except Exception: + pass + return _FallbackDestination(provider, base_url, api_mode, model) -def _fallback_entry_has_explicit_route(entry: Dict[str, Any]) -> bool: - """Return whether a fallback resolves outside its provider-owned route.""" - return _route_has_explicit_endpoint_or_key( - str(entry.get("base_url") or "").strip() or None, - _fallback_entry_api_key(entry), - ) +def _fallback_destination_from_entry( + entry: Dict[str, Any], + fb_client: Any, + fb_model: Optional[str], +) -> _FallbackDestination: + provider = str(entry.get("provider") or "").strip() + base_url = str( + entry.get("base_url") or getattr(fb_client, "base_url", "") or "" + ).strip() + api_mode = str( + entry.get("api_mode") or entry.get("transport") or "" + ).strip() or None + model = fb_model or str(entry.get("model") or "").strip() or None + return _complete_fallback_destination(provider, base_url, api_mode, model) -def _fallback_health_label( - fb_label: str, +def _fallback_destination( + task: Optional[str], fb_client: Any, - task: Optional[str] = None, -) -> str: - """Return the narrowest safe health key for a fallback candidate. - - A fallback entry with its own base URL or key is independently routed and - stays entry-scoped. A model-only entry shares its provider credentials and - endpoint with sibling entries, so a deployment/credential outage must mark - the provider-wide key. The built-in ``api-key`` label represents a discovery - bucket containing several independent providers; resolve it to the concrete - selected backend, or leave it unquarantined when the endpoint is unknown. - """ - label = str(fb_label or "").strip() - if label == "api-key": - return _recoverable_pool_provider("auto", fb_client) or "" + fb_model: Optional[str], + fb_label: str, +) -> _FallbackDestination: + """Return the resolved route identity used by a fallback request.""" + attached = getattr(fb_client, "_hermes_fallback_destination", None) + if isinstance(attached, _FallbackDestination): + return attached - entry = _fallback_entry_for_health(task, label) - if entry is None: - # Labels can also come from built-in fallbacks rather than config. Keep - # unknown shapes exact instead of risking a provider-wide quarantine. - return label - entry_scoped = _fallback_entry_has_explicit_route(entry) - if entry_scoped: - return label - return _normalize_aux_provider(_fallback_label_provider(label)) + provider = _fallback_provider_from_label(fb_label) + base_url = str(getattr(fb_client, "base_url", "") or "") + api_mode = None + model = fb_model + entry = _fallback_chain_entry(task, fb_label) + if entry is not None: + return _fallback_destination_from_entry(entry, fb_client, fb_model) -def _mark_recoverable_provider_unhealthy( - resolved_provider: str, - client: Any, - *, - main_runtime: Optional[Dict[str, Any]] = None, - task: Optional[str] = None, - route_is_explicit: bool = False, -) -> Optional[str]: - """Quarantine only a concrete shared provider inferred for this route.""" - route_label = str(resolved_provider or "").strip() - if re.match(r"fallback_(?:chain|providers)\[\d+\]", route_label): - health_provider = _fallback_health_label(route_label, client, task) - else: - health_provider = _recoverable_pool_provider( - resolved_provider, - client, - main_runtime=main_runtime, - route_is_explicit=route_is_explicit, - ) - if health_provider: - _mark_provider_unhealthy(health_provider) - return health_provider + return _complete_fallback_destination(provider, base_url, api_mode, model) -def _call_fallback_with_transient_retry_sync( - fb_client: Any, - fb_kwargs: dict, - task: Optional[str], - fb_label: str, -) -> Any: - """Call a fallback with the same bounded transient budget as the primary.""" - try: - return _validate_llm_response( - fb_client.chat.completions.create(**fb_kwargs), task) - except Exception as transient_err: - if not _is_transient_transport_error(transient_err): - raise - if task == "compression" and _is_timeout_error(transient_err): - raise - max_retries = _transient_retry_count() - last_transient = transient_err - for attempt in range(1, max_retries + 1): - backoff = min( - _TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (attempt - 1)), - 8.0, - ) - logger.info( - "Auxiliary %s: fallback candidate %s hit a transient error " - "(attempt %d/%d); retrying after %.1fs: %s", - task or "call", - fb_label, - attempt, - max_retries, - backoff, - last_transient, - ) - time.sleep(backoff) - try: - return _validate_llm_response( - fb_client.chat.completions.create(**fb_kwargs), task) - except Exception as retry_err: - if not _is_transient_transport_error(retry_err): - raise - last_transient = retry_err - raise last_transient - +def _replan_synchronous_cache_sections( + messages: list, + tools: Optional[list], + *, + destination: _FallbackDestination, +) -> tuple[list, list]: + """Strip source decoration and plan one synchronous destination locally.""" + from agent.agent_runtime_helpers import plan_cache_sections_for_destination -async def _call_fallback_with_transient_retry_async( - fb_client: Any, - fb_kwargs: dict, - task: Optional[str], - fb_label: str, -) -> Any: - """Async mirror of :func:`_call_fallback_with_transient_retry_sync`.""" - try: - return _validate_llm_response( - await fb_client.chat.completions.create(**fb_kwargs), task) - except Exception as transient_err: - if not _is_transient_transport_error(transient_err): - raise - if task == "compression" and _is_timeout_error(transient_err): - raise - max_retries = _transient_retry_count() - last_transient = transient_err - for attempt in range(1, max_retries + 1): - backoff = min( - _TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (attempt - 1)), - 8.0, - ) - logger.info( - "Auxiliary %s (async): fallback candidate %s hit a transient " - "error (attempt %d/%d); retrying after %.1fs: %s", - task or "call", - fb_label, - attempt, - max_retries, - backoff, - last_transient, - ) - await asyncio.sleep(backoff) - try: - return _validate_llm_response( - await fb_client.chat.completions.create(**fb_kwargs), task) - except Exception as retry_err: - if not _is_transient_transport_error(retry_err): - raise - last_transient = retry_err - raise last_transient + return plan_cache_sections_for_destination( + messages, + tools, + provider=destination.provider, + base_url=destination.base_url, + api_mode=destination.api_mode or "", + model=destination.model or "", + ) def _call_fallback_candidate_sync( @@ -4250,8 +4674,7 @@ def _call_fallback_candidate_sync( On an auth error: refresh the candidate's provider credentials and retry once with a rebuilt client; if the retry also auth-fails (non-refreshable expired token), mark the provider unhealthy and return ``None`` so the - caller can continue to the next fallback layer. Deployment-unavailable - errors are likewise quarantined and skipped. Other errors raise. + caller can continue to the next fallback layer. Non-auth errors raise. ``effective_timeout`` is the task-level deadline; a configured-chain candidate with its own ``timeout`` entry gets that instead, so a @@ -4266,86 +4689,81 @@ def _call_fallback_candidate_sync( task or "call", fb_label, fb_timeout, effective_timeout, ) effective_timeout = fb_timeout - fb_base = str(getattr(fb_client, "base_url", "") or "") - fb_provider = _fallback_label_provider(fb_label) + destination = _fallback_destination(task, fb_client, fb_model, fb_label) + fallback_messages, fallback_tools = _replan_synchronous_cache_sections( + messages, + tools, + destination=destination, + ) fb_kwargs = _build_call_kwargs( - fb_provider, fb_model, messages, + destination.provider, destination.model, fallback_messages, temperature=temperature, max_tokens=max_tokens, - tools=tools, timeout=effective_timeout, + tools=fallback_tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=fb_base, task=task) + base_url=destination.base_url, task=task) try: - return _call_fallback_with_transient_retry_sync( - fb_client, - fb_kwargs, + return _validate_llm_response( + _relay_sync_completion( + fb_client, + fb_kwargs, + provider=destination.provider, + api_mode=destination.api_mode, + ), task, - fb_label, ) except Exception as fb_err: - if _is_provider_deployment_unavailable_error(fb_err): - # Independently routed entries stay exact; model-only entries share - # their provider route and therefore quarantine that provider. - health_label = _fallback_health_label(fb_label, fb_client, task) - if health_label: - _mark_provider_unhealthy(health_label) - logger.warning( - "Auxiliary %s: fallback candidate %s is unavailable (%s) — " - "skipping to next fallback", - task or "call", fb_label, fb_err, - ) - return None if not _is_auth_error(fb_err): raise - refresh_provider = _auth_refresh_provider_for_route( - fb_provider, - fb_base, + fb_provider = _auth_refresh_provider_for_route( + destination.provider, destination.base_url ) - if refresh_provider not in {"auto", "", None} and _refresh_provider_credentials(refresh_provider): - retry_client, retry_model = _resolve_refreshed_fallback_candidate( - fb_label, - task, - refresh_provider, - fb_model, + if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider): + retry_client, retry_model = _get_cached_client( + fb_provider, + destination.model, + base_url=destination.base_url or None, + api_mode=destination.api_mode, ) if retry_client is not None: + retry_destination = _FallbackDestination( + fb_provider, + destination.base_url + or str(getattr(retry_client, "base_url", "") or ""), + destination.api_mode, + retry_model or destination.model, + ) + retry_messages, retry_tools = _replan_synchronous_cache_sections( + messages, + tools, + destination=retry_destination, + ) retry_kwargs = _build_call_kwargs( - refresh_provider, retry_model or fb_model, messages, + retry_destination.provider, + retry_destination.model, + retry_messages, temperature=temperature, max_tokens=max_tokens, - tools=tools, timeout=effective_timeout, + tools=retry_tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) + base_url=retry_destination.base_url, task=task) try: - return _call_fallback_with_transient_retry_sync( - retry_client, - retry_kwargs, + return _validate_llm_response( + _relay_sync_completion( + retry_client, + retry_kwargs, + provider=retry_destination.provider, + api_mode=retry_destination.api_mode, + ), task, - fb_label, ) except Exception as retry_err: - if _is_provider_deployment_unavailable_error(retry_err): - health_label = _fallback_health_label( - fb_label, - retry_client, - task, - ) - if health_label: - _mark_provider_unhealthy(health_label) - logger.warning( - "Auxiliary %s: refreshed fallback candidate %s is " - "unavailable (%s) — skipping to next fallback", - task or "call", fb_label, retry_err, - ) - return None if not _is_auth_error(retry_err): raise # Refresh unavailable or the refreshed credential still 401s — # the token is dead (expired setup token with no refresh token). # Quarantine the candidate so subsequent chain walks skip it, and # let the caller move on instead of aborting the whole task. - health_label = _fallback_health_label(fb_label, fb_client, task) - if health_label: - _mark_provider_unhealthy(health_label) + _mark_provider_unhealthy(fb_provider or fb_label) logger.warning( "Auxiliary %s: fallback candidate %s has a stale/unrefreshable " "credential (%s) — skipping to next fallback", @@ -4377,81 +4795,78 @@ async def _call_fallback_candidate_async( task or "call", fb_label, fb_timeout, effective_timeout, ) effective_timeout = fb_timeout - fb_base = str(getattr(fb_client, "base_url", "") or "") - fb_provider = _fallback_label_provider(fb_label) + destination = _fallback_destination(task, fb_client, fb_model, fb_label) + fallback_messages, fallback_tools = _replan_synchronous_cache_sections( + messages, + tools, + destination=destination, + ) fb_kwargs = _build_call_kwargs( - fb_provider, fb_model, messages, + destination.provider, destination.model, fallback_messages, temperature=temperature, max_tokens=max_tokens, - tools=tools, timeout=effective_timeout, + tools=fallback_tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=fb_base, task=task) + base_url=destination.base_url, task=task) try: - return await _call_fallback_with_transient_retry_async( - fb_client, - fb_kwargs, + return _validate_llm_response( + await _relay_async_completion( + fb_client, + fb_kwargs, + provider=destination.provider, + api_mode=destination.api_mode, + ), task, - fb_label, ) except Exception as fb_err: - if _is_provider_deployment_unavailable_error(fb_err): - health_label = _fallback_health_label(fb_label, fb_client, task) - if health_label: - _mark_provider_unhealthy(health_label) - logger.warning( - "Auxiliary %s (async): fallback candidate %s is unavailable " - "(%s) — skipping to next fallback", - task or "call", fb_label, fb_err, - ) - return None if not _is_auth_error(fb_err): raise - refresh_provider = _auth_refresh_provider_for_route( - fb_provider, - fb_base, + fb_provider = _auth_refresh_provider_for_route( + destination.provider, destination.base_url ) - if refresh_provider not in {"auto", "", None} and _refresh_provider_credentials(refresh_provider): - retry_client, retry_model = _resolve_refreshed_fallback_candidate( - fb_label, - task, - refresh_provider, - fb_model, + if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider): + retry_client, retry_model = _get_cached_client( + fb_provider, + destination.model, async_mode=True, + base_url=destination.base_url or None, + api_mode=destination.api_mode, ) if retry_client is not None: + retry_destination = _FallbackDestination( + fb_provider, + destination.base_url + or str(getattr(retry_client, "base_url", "") or ""), + destination.api_mode, + retry_model or destination.model, + ) + retry_messages, retry_tools = _replan_synchronous_cache_sections( + messages, + tools, + destination=retry_destination, + ) retry_kwargs = _build_call_kwargs( - refresh_provider, retry_model or fb_model, messages, + retry_destination.provider, + retry_destination.model, + retry_messages, temperature=temperature, max_tokens=max_tokens, - tools=tools, timeout=effective_timeout, + tools=retry_tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) + base_url=retry_destination.base_url, task=task) try: - return await _call_fallback_with_transient_retry_async( - retry_client, - retry_kwargs, + return _validate_llm_response( + await _relay_async_completion( + retry_client, + retry_kwargs, + provider=retry_destination.provider, + api_mode=retry_destination.api_mode, + ), task, - fb_label, ) except Exception as retry_err: - if _is_provider_deployment_unavailable_error(retry_err): - health_label = _fallback_health_label( - fb_label, - retry_client, - task, - ) - if health_label: - _mark_provider_unhealthy(health_label) - logger.warning( - "Auxiliary %s (async): refreshed fallback candidate " - "%s is unavailable (%s) — skipping to next fallback", - task or "call", fb_label, retry_err, - ) - return None if not _is_auth_error(retry_err): raise - health_label = _fallback_health_label(fb_label, fb_client, task) - if health_label: - _mark_provider_unhealthy(health_label) + _mark_provider_unhealthy(fb_provider or fb_label) logger.warning( "Auxiliary %s (async): fallback candidate %s has a stale/unrefreshable " "credential (%s) — skipping to next fallback", @@ -4474,14 +4889,13 @@ def _try_payment_fallback( (client, model, provider_label) or (None, None, "") if no fallback. """ # Normalise the failed provider label for matching. - skip = _normalize_aux_provider(failed_provider) + skip = failed_provider.lower().strip() # Also skip Step-1 main-provider path if it maps to the same backend. # (e.g. main_provider="openrouter" → skip "openrouter" in chain) main_provider = _read_main_provider() skip_labels = {skip} - main_norm = _normalize_aux_provider(main_provider) - if main_norm and main_norm == skip: - skip_labels.add(main_norm) + if main_provider and main_provider.lower() in skip: + skip_labels.add(main_provider.lower()) # Map common resolved_provider values back to chain labels. _alias_to_label = {"openrouter": "openrouter", "nous": "nous", "openai-codex": "openai-codex", "codex": "openai-codex", @@ -4516,6 +4930,7 @@ def _try_main_agent_model_fallback( failed_provider: str, task: str = None, reason: str = "error", + failed_model: Optional[str] = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Last-resort fallback to the user's main agent provider + model. @@ -4524,8 +4939,23 @@ def _try_main_agent_model_fallback( layer: if nothing the user asked for can serve the request, try the main chat model before giving up. - Skips when the failed provider already IS the main provider (no point - retrying the same backend that just failed). + ``failed_model`` narrows the same-provider skip to the exact + (provider, model) pair that just failed, mirroring + :func:`_try_configured_fallback_chain`. This matters for self-hosted / + custom endpoints serving several models behind one provider label: the + aux compression model timing out says nothing about the health of the + main agent model deployed on the same URL (real incident: aux + ``glm-5.2`` hung and timed out while main ``macaron-v1-venti`` on the + identical endpoint was serving 448K-token turns fine — the + provider-label skip discarded the one fallback that would have worked). + + - Model-specific runtime failures (timeout, connection, rate limit, + model-incompatible, invalid response) pass ``failed_model``: skip the + main model only when it IS the exact model that failed. + - Provider-wide failures (auth 401, payment 402) and legacy callers + leave ``failed_model`` as None, keeping the whole-provider skip — + the shared credentials/account are broken, so the main model on the + same provider cannot help either. Returns: (client, model, provider_label) or (None, None, "") if no fallback. @@ -4539,26 +4969,29 @@ def _try_main_agent_model_fallback( if not _agg_provider or not _agg_model: return None, None, "" main_provider, main_model = _agg_provider, _agg_model - main_norm = _normalize_aux_provider(main_provider) - if not main_provider or not main_model or main_norm in {"auto", ""}: + if not main_provider or not main_model or main_provider.lower() in {"auto", ""}: return None, None, "" - skip = _normalize_aux_provider(failed_provider) - if main_norm == skip: - # The thing that failed IS the main model — nothing to fall back to. - return None, None, "" - label = f"main-agent({main_provider})" - unhealthy_label = ( - label - if _is_provider_unhealthy(label) - else ( - main_norm - if _is_provider_unhealthy(main_norm) - else "" - ) + # Identity + scope semantics owned by agent.backend_identity (#72468): + # model-scoped failures skip only the exact deployment that failed; + # provider-wide failures (no failed_model) skip the credential surface. + from agent.backend_identity import ( + BackendIdentity, + FailureScope, + should_skip_candidate, ) - if unhealthy_label: - _log_skip_unhealthy(unhealthy_label, task) + + skip_model = (failed_model or "").strip().lower() or None + if should_skip_candidate( + BackendIdentity.build(provider=main_provider, model=main_model), + BackendIdentity.build(provider=failed_provider, model=skip_model), + FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL, + ): + # The thing that failed IS the main model (or the failure was + # provider-wide) — nothing to fall back to. + return None, None, "" + if _is_provider_unhealthy(main_provider): + _log_skip_unhealthy(main_provider, task) return None, None, "" try: @@ -4571,6 +5004,7 @@ def _try_main_agent_model_fallback( if client is None: return None, None, "" + label = f"main-agent({main_provider})" logger.info( "Auxiliary %s: %s on %s — falling back to main agent model %s (%s)", task or "call", reason, failed_provider, label, resolved_model or main_model, @@ -4664,6 +5098,7 @@ def _try_configured_fallback_chain( task: str, failed_provider: str, reason: str = "error", + failed_model: Optional[str] = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Try user-configured fallback_chain for a specific auxiliary task. @@ -4671,6 +5106,25 @@ def _try_configured_fallback_chain( entry in order. Each entry must have at least ``provider``; ``model``, ``base_url``, and ``api_key`` are optional. + ``failed_model`` narrows the skip check to the exact (provider, model) + pair that just failed, rather than the whole provider. Without it every + entry sharing the failed provider is skipped (the original behaviour). + Callers pass it only when a sibling model on the same provider could + plausibly recover: + + - Model-specific runtime failures (timeout, connection, rate limit, + model-incompatible, invalid response) pass ``failed_model`` so a + chain that intentionally lists several models under the same provider + — e.g. two more NVIDIA NIM models after the primary NIM model times + out — is not skipped wholesale. Only the exact model that failed is + skipped; the siblings still run instead of jumping straight to the + main-agent-model safety net. + - Provider-wide failures (auth 401, payment 402) and "no client could + be built" callers leave ``failed_model`` as None, keeping the whole + provider skipped — the shared credentials/account behind every model + on that provider are broken, so a sibling can't help and the + main-agent-model safety net should be reached instead. + Returns: (client, model, provider_label) or (None, None, "") if no fallback. """ @@ -4682,7 +5136,24 @@ def _try_configured_fallback_chain( if not chain or not isinstance(chain, list): return None, None, "" - skip = _normalize_aux_provider(failed_provider) + skip_model = (failed_model or "").strip().lower() or None + # Identity + scope semantics owned by agent.backend_identity (#59561, + # #72468): a failed_model means the failure was model-scoped (timeout / + # connection / rate limit) — only the exact deployment is skipped; no + # failed_model means provider-wide (auth/payment) — the whole credential + # surface is skipped. + from agent.backend_identity import ( + BackendIdentity, + FailureScope, + should_skip_candidate, + ) + + failed_ident = BackendIdentity.build( + provider=failed_provider, model=skip_model, + ) + failure_scope = ( + FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL + ) tried = [] min_ctx = _task_minimum_context_length(task) @@ -4692,26 +5163,20 @@ def _try_configured_fallback_chain( fb_provider = str(entry.get("provider", "")).strip() if not fb_provider: continue - fb_model = str(entry.get("model", "")).strip() or None - fb_norm = _normalize_aux_provider(fb_provider) - - label = f"fallback_chain[{i}]({fb_provider})" - entry_scoped = _fallback_entry_has_explicit_route(entry) - if fb_norm == skip and not entry_scoped: - continue - unhealthy_label = ( - label - if _is_provider_unhealthy(label) - else ( - fb_norm - if not entry_scoped and _is_provider_unhealthy(fb_norm) - else "" - ) - ) - if unhealthy_label: - _log_skip_unhealthy(unhealthy_label, task) - tried.append(f"{label} (unhealthy)") + fb_model_raw = str(entry.get("model", "")).strip() + if should_skip_candidate( + BackendIdentity.build( + provider=fb_provider, + model=fb_model_raw, + base_url=str(entry.get("base_url") or ""), + ), + failed_ident, + failure_scope, + ): continue + fb_model = fb_model_raw or None + + label = f"fallback_chain[{i}]({fb_provider})" try: fb_client, resolved_model = _resolve_fallback_entry(entry) @@ -4770,11 +5235,19 @@ def _try_configured_fallback_for_unavailable_client( ) -def _resolve_fallback_entry( - entry: Dict[str, Any], - *, - async_mode: bool = False, -) -> Tuple[Optional[Any], Optional[str]]: +def _fallback_entry_api_key(entry: Dict[str, Any]) -> Optional[str]: + """Resolve inline or env-backed API key from a fallback-chain entry. + + Delegates to the centralized, secret-scope-aware resolver so this path + doesn't leak another profile's credential via a raw ``os.getenv`` under + gateway multiplexing (see ``hermes_cli.fallback_config.resolve_entry_api_key``). + """ + from hermes_cli.fallback_config import resolve_entry_api_key + + return resolve_entry_api_key(entry) + + +def _resolve_fallback_entry(entry: Dict[str, Any]) -> Tuple[Optional[Any], Optional[str]]: """Resolve one fallback entry through the central provider router.""" provider = str(entry.get("provider") or "").strip() model = str(entry.get("model") or "").strip() or None @@ -4783,39 +5256,21 @@ def _resolve_fallback_entry( base_url = str(entry.get("base_url") or "").strip() or None api_key = _fallback_entry_api_key(entry) api_mode = str(entry.get("api_mode") or entry.get("transport") or "").strip() or None - return resolve_provider_client( + client, resolved_model = resolve_provider_client( provider, model=model, - async_mode=async_mode, explicit_base_url=base_url, explicit_api_key=api_key, api_mode=api_mode, ) - - -def _resolve_refreshed_fallback_candidate( - fb_label: str, - task: Optional[str], - refresh_provider: str, - fb_model: Optional[str], - *, - async_mode: bool = False, -) -> Tuple[Optional[Any], Optional[str]]: - """Rebuild a refreshed fallback without losing entry-scoped routing.""" - entry = _fallback_entry_for_health(task, fb_label) - entry_scoped = isinstance(entry, dict) and any( - str(entry.get(field) or "").strip() - for field in ("base_url", "api_key", "key_env", "api_key_env", "api_mode", "transport") - ) - if entry_scoped: - return _resolve_fallback_entry(entry, async_mode=async_mode) - if async_mode: - return _get_cached_client( - refresh_provider, - fb_model, - async_mode=True, - ) - return _get_cached_client(refresh_provider, fb_model) + if client is not None: + try: + client._hermes_fallback_destination = _fallback_destination_from_entry( + entry, client, resolved_model + ) + except Exception: + pass + return client, resolved_model def _try_main_fallback_chain( @@ -4832,10 +5287,10 @@ def _try_main_fallback_chain( participate in the same order as the main agent. """ try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly from hermes_cli.fallback_config import get_fallback_chain - chain = get_fallback_chain(load_config()) + chain = get_fallback_chain(load_config_readonly()) except Exception as exc: logger.debug("Auxiliary %s: could not load main fallback chain: %s", task or "call", exc) return None, None, "" @@ -4843,8 +5298,8 @@ def _try_main_fallback_chain( if not chain: return None, None, "" - failed_norm = _normalize_aux_provider(failed_provider) - main_norm = _normalize_aux_provider(_read_main_provider()) + failed_norm = (failed_provider or "").strip().lower() + main_norm = (_read_main_provider() or "").strip().lower() skip = {p for p in (failed_norm, main_norm, "auto") if p} tried: List[str] = [] min_ctx = _task_minimum_context_length(task) @@ -4856,17 +5311,12 @@ def _try_main_fallback_chain( fb_model = str(entry.get("model") or "").strip() if not fb_provider or not fb_model: continue - fb_norm = _normalize_aux_provider(fb_provider) + fb_norm = fb_provider.lower() label = f"fallback_providers[{i}]({fb_provider})" if fb_norm in skip: tried.append(f"{label} (skipped)") continue - if _is_provider_unhealthy(label): - _log_skip_unhealthy(label, task) - tried.append(f"{label} (unhealthy)") - continue - entry_scoped = _fallback_entry_has_explicit_route(entry) - if not entry_scoped and _is_provider_unhealthy(fb_norm): + if _is_provider_unhealthy(fb_norm): _log_skip_unhealthy(fb_norm, task) tried.append(f"{label} (unhealthy)") continue @@ -4895,7 +5345,7 @@ def _try_main_fallback_chain( task or "call", reason, failed_provider or "auto", label, resolved_model or fb_model, ) - return fb_client, resolved_model or fb_model, label + return fb_client, resolved_model or fb_model, fb_provider tried.append(label) if tried: @@ -5173,6 +5623,10 @@ def _to_async_client(sync_client, model: str, is_vision: bool = False): async_kwargs["default_headers"] = {"User-Agent": "claude-code/0.1.0"} elif base_url_host_matches(sync_base_url, "integrate.api.nvidia.com"): async_kwargs["default_headers"] = build_nvidia_nim_headers(sync_base_url) + elif base_url_host_matches(sync_base_url, "x.ai"): + from tools.xai_http import hermes_xai_default_headers + + async_kwargs["default_headers"] = hermes_xai_default_headers() else: # Fall back to profile.default_headers for providers that declare # client-level headers on their ProviderProfile (e.g. attribution @@ -5324,7 +5778,16 @@ def resolve_provider_client( # sent to Codex after the main lane fell back to gpt-5.5). Let _resolve_auto() # return the actual current runtime model when the caller did not explicitly # request one. (# compression-current-model) - if not model and provider != "auto": + # + # Nous + vision is the one carve-out: the branch below resolves its model + # from the Portal's tier-aware vision recommendation (``_try_nous(vision= + # True)``), and ``final_model = model or default`` means anything pre-filled + # here wins over that. The main chat model is routinely text-only (e.g. a + # ``:free`` chat SKU), so pre-filling it sends the image to a model that + # cannot accept one and the Portal 404s. Leave ``model`` unset and let the + # Portal slot through; only an explicit caller model may override it. + _nous_portal_vision = provider == "nous" and is_vision + if not model and provider != "auto" and not _nous_portal_vision: model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model def _needs_codex_wrap(client_obj, base_url_str: str, model_str: str) -> bool: @@ -5409,10 +5872,11 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", # ── Nous Portal (OAuth) ────────────────────────────────────────── if provider == "nous": - # Detect vision tasks: either explicit model override from - # _PROVIDER_VISION_MODELS, or caller passed a known vision model. + # Detect vision tasks: caller flag (strict vision backend), explicit + # model override from _PROVIDER_VISION_MODELS, or a known vision id. _is_vision = ( - model in _PROVIDER_VISION_MODELS.values() + is_vision + or model in _PROVIDER_VISION_MODELS.values() or (model or "").strip().lower() == "mimo-v2-omni" ) client, default = _try_nous(vision=_is_vision) @@ -5421,6 +5885,17 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", "but Nous Portal not configured (run: hermes auth)") return None, None final_model = _normalize_resolved_model(model or default, provider) + # Dual-wire: anthropic/* → /v1/messages, everything else stays on + # /chat/completions. Derive from the catalog id (not a stale + # api_mode=chat_completions) so aux matches the main agent. + from hermes_cli.providers import nous_api_mode + + portal_mode = nous_api_mode(final_model) + api_key_str = str(getattr(client, "api_key", "") or "") + base_url_str = str(getattr(client, "base_url", "") or "") + client = _maybe_wrap_anthropic( + client, final_model, api_key_str, base_url_str, portal_mode, + ) return (_to_async_client(client, final_model, is_vision=is_vision) if async_mode else (client, final_model)) @@ -5486,7 +5961,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", custom_base = _to_openai_base_url(explicit_base_url).strip() custom_key = ( (explicit_api_key or "").strip() - or os.getenv("OPENAI_API_KEY", "").strip() + or _scoped_key_env("OPENAI_API_KEY") or _read_main_api_key_if_same_host(custom_base) or "no-key-required" # local servers don't need auth ) @@ -5582,7 +6057,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", custom_key = (custom_entry.get("api_key") or "").strip() custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip() if not custom_key and custom_key_env: - custom_key = os.getenv(custom_key_env, "").strip() + custom_key = _scoped_key_env(custom_key_env) custom_key = custom_key or "no-key-required" if custom_key == "no-key-required": logger.warning( @@ -5784,10 +6259,15 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", )) elif base_url_host_matches(base_url, "integrate.api.nvidia.com"): headers.update(build_nvidia_nim_headers(base_url)) + elif base_url_host_matches(base_url, "x.ai"): + from tools.xai_http import hermes_xai_default_headers + + headers.update(hermes_xai_default_headers()) else: # Fall back to profile.default_headers for providers that declare # client-level attribution headers on their profile (e.g. GMI - # User-Agent for traffic identification). + # User-Agent for traffic identification, Vercel AI Gateway + # Referer/Title for analytics). try: from providers import get_provider_profile as _gpf_main _ph_main = _gpf_main(provider) @@ -5872,9 +6352,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", args = list(creds.get("args") or []) # `agy` currently exposes no model-selection flag. Do not inherit # the main Hermes model here; that would falsely claim a GPT/Claude - # model handled Antigravity output. Hardcoded because the universal - # fallback at the top of resolve_provider_client() pre-fills `model` - # with the user's main model. + # model handled Antigravity output. final_model = "antigravity-cli" from agent.google_antigravity_cli_adapter import GoogleAntigravityCLIClient @@ -6071,11 +6549,11 @@ def _main_model_supports_vision(provider: str, model: Optional[str]) -> bool: """ try: from agent.image_routing import _lookup_supports_vision - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly except ImportError: return True try: - supports = _lookup_supports_vision(provider, model, load_config()) + supports = _lookup_supports_vision(provider, model, load_config_readonly()) except Exception: # pragma: no cover - defensive return True if supports is None: @@ -6100,7 +6578,10 @@ def _resolve_strict_vision_backend( if provider == "openrouter": return _try_openrouter(model=model) if provider == "nous": - return _try_nous(vision=True) + # Must go through resolve_provider_client so anthropic/* vision + # recommendations wrap onto /v1/messages — _try_nous alone returns + # a bare OpenAI client and the call 404s. + return resolve_provider_client("nous", model, is_vision=True) if provider == "openai-codex": # Route through resolve_provider_client so the caller's explicit # model is used. There is no safe default Codex model (shifting @@ -6250,10 +6731,17 @@ def _finalize(resolved_provider: str, sync_client: Any, default_model: Optional[ # DeepSeek-V4-Flash default) and _main_model_supports_vision can't be # trusted to catch that. Only fall back to the chat model when no # provider default is available (catalog unreachable). - vision_model = _resolve_provider_vision_default(main_provider) or main_model + provider_vision_default = _resolve_provider_vision_default(main_provider) + vision_model = provider_vision_default or main_model if main_provider == "nous": + # Nous resolves its vision model from the Portal's tier-aware + # recommended-models slots inside _try_nous(vision=True). + # Passing the chat model here overrides that pick, so a + # text-only chat default (e.g. a `:free` chat SKU) receives the + # image and the upstream rejects it with a 404. Only an + # explicit auxiliary.vision.model may override the Portal. sync_client, default_model = _resolve_strict_vision_backend( - main_provider, vision_model + main_provider, resolved_model or provider_vision_default ) if sync_client is not None: logger.info( @@ -6412,7 +6900,7 @@ def auxiliary_max_tokens_param(value: int, *, model: Optional[str] = None) -> di misses the case where a custom base URL serves e.g. ``gpt-5.4``. """ custom_base = _current_custom_base_url() - or_key = os.getenv("OPENROUTER_API_KEY") + or_key = _scoped_key_env("OPENROUTER_API_KEY") # Use max_completion_tokens for direct OpenAI-compatible providers that reject # max_tokens on newer GPT-4o/o-series/GPT-5-style models. _custom_host = base_url_hostname(custom_base) or "" @@ -6873,7 +7361,7 @@ def _resolve_task_provider_model( task_config.get("key_env") or task_config.get("api_key_env") or "" ).strip() if cfg_key_env: - cfg_api_key = os.getenv(cfg_key_env, "").strip() or None + cfg_api_key = _scoped_key_env(cfg_key_env) or None cfg_api_mode = str(task_config.get("api_mode", "")).strip() or None # 'auto' is a sentinel meaning "inherit from main runtime / auto-detect", not @@ -7041,8 +7529,8 @@ def _get_auxiliary_task_config(task: str) -> Dict[str, Any]: if not task: return {} try: - from hermes_cli.config import load_config - config = load_config() + from hermes_cli.config import load_config_readonly + config = load_config_readonly() except ImportError: return {} aux = config.get("auxiliary", {}) if isinstance(config, dict) else {} @@ -7361,8 +7849,14 @@ def _build_call_kwargs( _is_gemini_native = is_native_gemini_base_url(_effective_base) except Exception: pass + _nous_on_messages = False + if _provider_norm in {"nous", "nous-portal", "nousresearch"}: + from hermes_cli.providers import nous_api_mode + + _nous_on_messages = nous_api_mode(model) == "anthropic_messages" if ( _is_anthropic_compat_endpoint(provider, _effective_base) + or _nous_on_messages or _is_nvidia_nim or _is_moa or _is_gemini_native @@ -7457,21 +7951,43 @@ def _build_call_kwargs( else: effort = reasoning_config.get("effort") or "medium" merged_extra["reasoning"] = {"enabled": True, "effort": effort} - if provider == "nous" and "tags" not in merged_extra: - merged_extra["tags"] = _nous_portal_tags() + # Portal product tags + sticky session_id. The provider profile usually + # supplies both; this fallback covers profile-load failures and alias + # spellings the profile lookup might miss. session_id keeps aux + # compression/title/vision calls on the same upstream instance as the + # main turn (cache warmth) — tags alone are not enough on /v1/messages. + _provider_for_portal = str(provider or "").strip().lower() + if _provider_for_portal in {"nous", "nous-portal", "nousresearch"}: + if "tags" not in merged_extra: + merged_extra["tags"] = _nous_portal_tags() + if "session_id" not in merged_extra: + try: + from agent.portal_tags import get_conversation_context + + sticky_key = get_conversation_context() + except Exception: + sticky_key = None + if sticky_key: + merged_extra["session_id"] = sticky_key if merged_extra: kwargs["extra_body"] = merged_extra - # Native Anthropic Messages adapters do not consume ``extra_body``. Carry - # the normalized Hermes reasoning config through a private kwarg so the - # adapter can pass it into build_anthropic_kwargs(), where provider-aware - # thinking/output_config projection lives. Do not expose this private kwarg - # to ordinary OpenAI-compatible SDK clients, which would reject it. + # Anthropic Messages adapters translate Hermes reasoning into native + # ``thinking`` via a private kwarg (and strip OpenAI-shaped + # ``extra_body.reasoning``). Do not expose this private kwarg to ordinary + # OpenAI-compatible SDK clients, which would reject it. Portal Claude is + # dual-wire — include it when the catalog id selects /v1/messages. if reasoning_config and isinstance(reasoning_config, dict): provider_norm = str(provider or "").strip().lower() effective_base = base_url or "" + _nous_on_messages = False + if provider_norm in {"nous", "nous-portal", "nousresearch"}: + from hermes_cli.providers import nous_api_mode + + _nous_on_messages = nous_api_mode(model) == "anthropic_messages" if ( provider_norm == "anthropic" + or _nous_on_messages or _endpoint_speaks_anthropic_messages(effective_base) or _is_anthropic_compat_endpoint(provider_norm, effective_base) ): @@ -7517,6 +8033,7 @@ def _validate_llm_response( except (AttributeError, TypeError, IndexError) as exc: recovered = _recover_aux_response_message(response) if recovered is not None: + _complete_relay_auxiliary_call() return recovered response_type = type(response).__name__ response_preview = str(response)[:120] @@ -7526,9 +8043,34 @@ def _validate_llm_response( f"Expected object with .choices[0].message — check provider " f"adapter or custom endpoint compatibility." ) from exc + _complete_relay_auxiliary_call() return response +def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None: + """Close one auxiliary logical call after acceptance or terminal failure.""" + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + from agent import relay_llm + + relay_llm.complete_logical_call( + str(context.get("request_id") or ""), + outcome=outcome, + ) + + +def _fail_relay_auxiliary_call() -> None: + """Close a terminally failed call without replacing its original error.""" + try: + _complete_relay_auxiliary_call(outcome="failed") + except Exception: + logger.warning( + "Relay auxiliary failure finalization failed", + exc_info=True, + ) + + def _recover_aux_response_message(response: Any) -> Optional[Any]: """Synthesize chat-completions shape from Responses-style text fields. @@ -7587,6 +8129,347 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any: return value +# ── Streamed aggregation for progress-hooked auxiliary calls ───────────── +# When a forward-progress hook is installed (aux_progress_hook — today only +# by context compression), the primary chat.completions attempt is upgraded +# to a streamed request that is aggregated back into a complete response. +# Two effects, both deliberate: +# 1. The configured ``timeout`` becomes an INTER-CHUNK idle timeout instead +# of a total budget (httpx applies the read timeout per stream read), so +# a slow-but-generating summary model is never killed mid-generation +# while tokens are moving — only a genuinely silent connection dies. +# 2. Every arriving chunk ticks the progress hook, letting outer watchdogs +# (gateway session hygiene) extend their deadlines on liveness instead +# of guessing with a fixed wall clock. +# A total ceiling still bounds the pathological 1-token-per-idle-window +# stream; see _aux_stream_total_ceiling(). + +_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0 +_AUX_STREAM_CEILING_MULTIPLIER = 4.0 + + +def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float: + """Absolute wall-clock bound for a progress-hooked streamed aux call. + + Generous by design — the idle timeout is the real guard; this only stops + a degenerate stream that trickles one token per idle window forever. + """ + try: + timeout = float(effective_timeout) if effective_timeout is not None else 0.0 + except (TypeError, ValueError): + timeout = 0.0 + return max(_AUX_STREAM_CEILING_FLOOR_SECONDS, + _AUX_STREAM_CEILING_MULTIPLIER * timeout) + + +def _client_streams_internally(client: Any) -> bool: + """Wire adapters that consume a stream inside .create() already tick the + progress hook themselves (Codex per SSE event, Anthropic per stream + event); Bedrock's Converse shim cannot stream at all. None of them + accept chat-completions ``stream=True`` semantics from us.""" + return isinstance(client, ( + CodexAuxiliaryClient, + AnthropicAuxiliaryClient, + BedrockAuxiliaryClient, + )) + + +def _is_streaming_rejected_error(exc: Exception) -> bool: + """Provider explicitly refused a streamed chat.completions request.""" + err = str(exc).lower() + if "stream_options" in err: + return True + return "stream" in err and ( + "not supported" in err + or "unsupported" in err + or "not allowed" in err + or "disabled" in err + ) + + +def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool: + """Detect providers that only accept streaming (non-stream = HTTP 400). + + Some OpenAI-compatible endpoints reject non-streaming chat requests + outright — e.g. Tencent Copilot returns + ``{"code": 11101, "msg": "Non-stream chat request is currently not + supported"}``. The main conversation loop already streams, so interactive + chat works; auxiliary tasks (title generation, compression, web extract) + used the non-streaming path and failed on every call. When this returns + True the auxiliary client sends ``stream=True`` and aggregates the chunks + itself (see :func:`_aggregate_chat_stream`). Credit @kudi88 (PR #60686). + + Beyond the known-host list, users can mark ANY custom endpoint as + stream-only via ``auxiliary.stream_only_base_urls`` in config.yaml + (list of substrings matched against the endpoint URL). + """ + _url = str(base_url or "").lower() + if not _url: + return False + # Tencent Copilot — "Non-stream chat request is currently not supported" + if base_url_host_matches(_url, "copilot.tencent.com"): + return True + try: + from hermes_cli.config import load_config + aux_cfg = (load_config() or {}).get("auxiliary", {}) + markers = aux_cfg.get("stream_only_base_urls") or [] + if isinstance(markers, (list, tuple)): + for marker in markers: + if isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url: + return True + except Exception: + # Config read is best-effort; never break an aux call over it. + pass + return False + + +def _create_with_progress( + client: Any, + kwargs: Dict[str, Any], + task: Optional[str] = None, + *, + force_stream: bool = False, +) -> Any: + """chat.completions.create() that streams when a progress hook is active + or the provider only accepts streamed requests. + + Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when + neither trigger applies (every existing caller/task) or when the client's + wire adapter streams internally. With a hook + a chunk-capable client, + the request is sent with ``stream=True`` and aggregated, ticking the hook + per chunk — so the configured ``timeout`` acts per stream read (idle) + rather than as a total budget, and outer liveness watchdogs see tokens + moving. ``force_stream=True`` (stream-only providers such as Tencent + Copilot — credit @kudi88, PR #60686) takes the same streamed path even + without a hook. Providers that reject the streamed request fall back to + the plain non-streaming call — except under ``force_stream``, where a + stream-only provider rejects the plain call by definition, so the + original error is surfaced to the normal recovery chains instead. + """ + _notify_aux_progress() # request dispatched counts as progress + if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client): + return client.chat.completions.create(**kwargs) + + total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout")) + stream_kwargs = dict(kwargs) + stream_kwargs["stream"] = True + stream_kwargs["stream_options"] = {"include_usage": True} + try: + chunks = client.chat.completions.create(**stream_kwargs) + except Exception as exc: + # Genuine provider failures (auth, credit, rate limit, network) are + # not streaming's fault — surface them unchanged so the existing + # recovery chains (credential refresh, pool rotation, provider + # fallback) see the same error they would on a plain call. + if ( + force_stream + or _is_transient_transport_error(exc) + or _is_auth_error(exc) + or _is_payment_error(exc) + or _is_rate_limit_error(exc) + ): + raise + # Anything else may be a streaming-specific rejection (explicit + # "stream not supported", stream_options 400, or an idiosyncratic + # 4xx). Retry non-streaming once; if the request itself is bad the + # plain call reproduces the real error for the normal except-chains. + logger.debug( + "Auxiliary %s: streamed request failed (%s); retrying " + "non-streaming", task or "call", exc, + ) + return client.chat.completions.create(**kwargs) + + # Some shims (MoA virtual provider under quiet mode, defensive adapters) + # return a complete response even when stream=True was requested. + if hasattr(chunks, "choices"): + _notify_aux_progress() + return chunks + return _aggregate_chat_stream( + chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling, + ) + + +def _aggregate_chat_stream( + chunks: Any, + *, + model: str = "", + total_ceiling: Optional[float] = None, +) -> Any: + """Consume a chat.completions chunk stream into a complete response. + + Ticks the thread-local aux progress hook on every chunk. Raises + TimeoutError when *total_ceiling* seconds elapse before the stream + finishes — phrased with "timed out" so existing timeout classification + (``_is_timeout_error``) treats it exactly like a request timeout. + Accumulation is shared with the async mirror via + :class:`_ChatStreamAccumulator`. + """ + acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + try: + for chunk in chunks: + acc.feed(chunk) + finally: + close_fn = getattr(chunks, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + pass + return acc.finish() + + +class _ChatStreamAccumulator: + """Shared per-chunk accumulation for sync and async stream aggregation. + + Mirrors :func:`_aggregate_chat_stream`'s chunk handling so the async + consumer below cannot drift from the sync one (same content/reasoning/ + tool-call delta reassembly, same "timed out" ceiling phrasing). + """ + + def __init__(self, model: str = "", total_ceiling: Optional[float] = None): + self._started = time.monotonic() + self._total_ceiling = total_ceiling + self.content_parts: List[str] = [] + self.reasoning_parts: List[str] = [] + self.tool_calls_acc: Dict[int, Dict[str, Any]] = {} + self.finish_reason = None + self.usage = None + self.resp_id = "" + self.resp_model = model or "" + + def feed(self, chunk: Any) -> None: + _notify_aux_progress() + if ( + self._total_ceiling is not None + and (time.monotonic() - self._started) >= self._total_ceiling + ): + raise TimeoutError( + f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s " + "total ceiling (stream still open but over budget)" + ) + self.resp_id = getattr(chunk, "id", None) or self.resp_id + self.resp_model = getattr(chunk, "model", None) or self.resp_model + chunk_usage = getattr(chunk, "usage", None) + if chunk_usage: + self.usage = chunk_usage + choices = getattr(chunk, "choices", None) or [] + if not choices: + return + choice = choices[0] + self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason + delta = getattr(choice, "delta", None) + if delta is None: + return + piece = getattr(delta, "content", None) + if piece: + self.content_parts.append(piece) + reasoning_piece = ( + getattr(delta, "reasoning", None) + or getattr(delta, "reasoning_content", None) + ) + if reasoning_piece and isinstance(reasoning_piece, str): + self.reasoning_parts.append(reasoning_piece) + for tc in (getattr(delta, "tool_calls", None) or []): + idx = getattr(tc, "index", 0) or 0 + acc = self.tool_calls_acc.setdefault( + idx, {"id": "", "name": "", "arguments": []} + ) + if getattr(tc, "id", None): + acc["id"] = tc.id + fn = getattr(tc, "function", None) + if fn is not None: + if getattr(fn, "name", None): + acc["name"] = fn.name + if getattr(fn, "arguments", None): + acc["arguments"].append(fn.arguments) + + def finish(self) -> Any: + tool_calls = None + if self.tool_calls_acc: + tool_calls = [ + SimpleNamespace( + id=acc["id"], + type="function", + function=SimpleNamespace( + name=acc["name"], + arguments="".join(acc["arguments"]), + ), + ) + for _idx, acc in sorted(self.tool_calls_acc.items()) + ] + message = SimpleNamespace( + role="assistant", + content="".join(self.content_parts), + tool_calls=tool_calls, + reasoning="".join(self.reasoning_parts) or None, + ) + choice = SimpleNamespace( + index=0, + message=message, + finish_reason=self.finish_reason or "stop", + ) + return SimpleNamespace( + id=self.resp_id, + model=self.resp_model, + object="chat.completion", + choices=[choice], + usage=self.usage, + ) + + +async def _aggregate_chat_stream_async( + chunks: Any, + *, + model: str = "", + total_ceiling: Optional[float] = None, +) -> Any: + """Async mirror of :func:`_aggregate_chat_stream` (``async for`` consumer). + + The AsyncOpenAI stream contract is an async iterator — consuming it with + the sync helper raises. Same accumulation and ceiling semantics via + :class:`_ChatStreamAccumulator`. + """ + acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + try: + async for chunk in chunks: + acc.feed(chunk) + finally: + close_fn = getattr(chunks, "close", None) or getattr(chunks, "aclose", None) + if callable(close_fn): + try: + result = close_fn() + if inspect.isawaitable(result): + await result + except Exception: + pass + return acc.finish() + + +async def _acreate_with_stream( + client: Any, + kwargs: Dict[str, Any], + task: Optional[str] = None, +) -> Any: + """Async chat.completions.create() for stream-only providers. + + Sends ``stream=True`` and aggregates the async chunk stream into a + complete response (credit @kudi88, PR #60686 — async contract fixed to + ``async for`` and tool-call deltas preserved per sweeper review). + """ + total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout")) + stream_kwargs = dict(kwargs) + stream_kwargs["stream"] = True + stream_kwargs["stream_options"] = {"include_usage": True} + chunks = await client.chat.completions.create(**stream_kwargs) + # Defensive: shims may hand back a complete response despite stream=True. + if hasattr(chunks, "choices"): + return chunks + return await _aggregate_chat_stream_async( + chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling, + ) + + +@_relay_auxiliary_call def call_llm( task: str = None, *, @@ -7727,6 +8610,11 @@ def call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Log what we're about to do — makes auxiliary operations visible _base_info = str(getattr(client, "base_url", resolved_base_url) or "") @@ -7764,7 +8652,22 @@ def call_llm( kwargs["stream"] = True if stream_options: kwargs["stream_options"] = stream_options - return client.chat.completions.create(**kwargs) + if task == "moa_aggregator" and isinstance(client, CodexAuxiliaryClient): + # CodexAuxiliaryClient (openai-codex, xai-oauth, and any other + # Responses-shim provider) consumes the provider stream internally + # and returns a completed response object. Routing that nested + # MoA stream through Relay's generic managed stream makes the + # manager iterate the completed SimpleNamespace itself (#55933). + # Return the provider call directly; the MoA facade converts a + # completed response into a one-chunk delta iterator at its + # boundary. + return client.chat.completions.create(**kwargs) + return _relay_sync_stream( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ) # Handle unsupported temperature, max_tokens vs max_completion_tokens retry, # then payment fallback. @@ -7787,7 +8690,21 @@ def call_llm( # for the transient retry every auxiliary task shares. (PR #16587) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task, + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, _base_info or resolved_base_url, + ), + ), + ), + task, provider=resolved_provider, base_url=_base_info) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -7820,7 +8737,22 @@ def call_llm( time.sleep(_backoff) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=lambda request: _create_with_progress( + client, + request, + task, + force_stream=_provider_requires_stream( + resolved_provider, + _base_info or resolved_base_url, + ), + ), + ), + task) except Exception as retry_transient: if not _is_transient_transport_error(retry_transient): raise @@ -7837,7 +8769,12 @@ def call_llm( ) try: return _validate_llm_response( - client.chat.completions.create(**retry_kwargs), task) + _relay_sync_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) # If retry still fails, fall through to the max_tokens / @@ -7848,7 +8785,6 @@ def call_llm( _is_payment_error(retry_err) or _is_connection_error(retry_err) or _is_auth_error(retry_err) - or _is_provider_deployment_unavailable_error(retry_err) or "max_tokens" in retry_err_str or "unsupported_parameter" in retry_err_str ): @@ -7876,16 +8812,16 @@ def call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. - if not ( - _is_payment_error(retry_err) - or _is_connection_error(retry_err) - or _is_rate_limit_error(retry_err) - or _is_provider_deployment_unavailable_error(retry_err) - ): + if not (_is_payment_error(retry_err) or _is_connection_error(retry_err) or _is_rate_limit_error(retry_err)): raise first_err = retry_err @@ -7911,7 +8847,12 @@ def call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -7944,7 +8885,12 @@ def call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -7972,7 +8918,12 @@ def call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - refreshed_client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry ─────────────────────────────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -8009,15 +8960,7 @@ def call_llm( ) # ── Same-provider credential-pool recovery ───────────────────── - pool_provider = _recoverable_pool_provider( - resolved_provider, - client, - main_runtime=main_runtime, - route_is_explicit=_route_has_explicit_endpoint_or_key( - resolved_base_url, - resolved_api_key, - ), - ) + pool_provider = _recoverable_pool_provider(resolved_provider, client, main_runtime=main_runtime) # Capture the exact API key used so mark_exhausted_and_rotate can find # the correct pool entry even when another process rotated the pool # between this call and recovery (which leaves current()=None and makes @@ -8030,7 +8973,12 @@ def call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _relay_sync_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise @@ -8103,7 +9051,6 @@ def call_llm( or _is_connection_error(first_err) or _is_rate_limit_error(first_err) or _is_model_incompatible_error(first_err) - or _is_provider_deployment_unavailable_error(first_err) or _is_invalid_aux_response_error(first_err) ) # Respect explicit provider choice for transient errors (auth, request @@ -8127,7 +9074,6 @@ def call_llm( or _is_connection_error(first_err) or _is_rate_limit_error(first_err) or _is_model_incompatible_error(first_err) - or _is_provider_deployment_unavailable_error(first_err) or _is_invalid_aux_response_error(first_err) ) if should_fallback and (is_auto or is_capacity_error): @@ -8139,32 +9085,13 @@ def call_llm( # "auto"; the client's base_url tells us which backend got the # 402). Mark THAT label unhealthy so subsequent aux calls # skip it instead of paying another doomed RTT. - _mark_recoverable_provider_unhealthy( - resolved_provider, - client, - main_runtime=main_runtime, - task=task, - route_is_explicit=_route_has_explicit_endpoint_or_key( - resolved_base_url, - resolved_api_key, - ), + _mark_provider_unhealthy( + _recoverable_pool_provider(resolved_provider, client, main_runtime=main_runtime) or resolved_provider ) elif _is_rate_limit_error(first_err): reason = "rate limit" elif _is_model_incompatible_error(first_err): reason = "model incompatible with route" - elif _is_provider_deployment_unavailable_error(first_err): - reason = "provider deployment unavailable" - _mark_recoverable_provider_unhealthy( - resolved_provider, - client, - main_runtime=main_runtime, - task=task, - route_is_explicit=_route_has_explicit_endpoint_or_key( - resolved_base_url, - resolved_api_key, - ), - ) elif _is_invalid_aux_response_error(first_err): reason = "invalid provider response" else: @@ -8172,42 +9099,41 @@ def call_llm( logger.info("Auxiliary %s: %s on %s (%s), trying fallback", task or "call", reason, resolved_provider, first_err) + # Narrow the configured-chain skip to the exact model that + # failed ONLY for model-specific failures. Auth (401) and + # payment (402) errors are provider-wide — the credentials or + # account behind every model on that provider are the same — so + # a sibling model can't recover; keep skipping the whole + # provider so the main-agent-model safety net is still reached. + _chain_failed_model = ( + None if reason in ("auth error", "payment error") else final_model + ) # Fallback order (#26882, #26803): # 1. User-configured fallback_chain (per-task) if set # 2. For auto: top-level main fallback_providers/fallback_model # 3. For auto: built-in auxiliary discovery chain # 4. For explicit aux providers: main agent model safety net - attempted_fallbacks: set[tuple[str, Optional[str]]] = set() - fallback_reason = reason - while True: - fb_client, fb_model, fb_label = (None, None, "") - if is_auto: - fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=fallback_reason) - if fb_client is None: - fb_client, fb_model, fb_label = _try_main_fallback_chain( - task, resolved_provider or "auto", reason=fallback_reason) - if fb_client is None: - fb_client, fb_model, fb_label = _try_payment_fallback( - resolved_provider, task, reason=fallback_reason) - else: - fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=fallback_reason) - if fb_client is None: - fb_client, fb_model, fb_label = _try_main_agent_model_fallback( - resolved_provider, task, reason=fallback_reason) - + fb_client, fb_model, fb_label = (None, None, "") + if is_auto: + fb_client, fb_model, fb_label = _try_configured_fallback_chain( + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) if fb_client is None: - break - fallback_identity = (fb_label, fb_model) - if fallback_identity in attempted_fallbacks: - logger.warning( - "Auxiliary %s: fallback selector repeated %s; stopping " - "to avoid a retry loop", - task or "call", fb_label, - ) - break - attempted_fallbacks.add(fallback_identity) + fb_client, fb_model, fb_label = _try_main_fallback_chain( + task, resolved_provider or "auto", reason=reason) + if fb_client is None: + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason=reason) + else: + fb_client, fb_model, fb_label = _try_configured_fallback_chain( + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) + if fb_client is None: + fb_client, fb_model, fb_label = _try_main_agent_model_fallback( + resolved_provider, task, reason=reason, + failed_model=_chain_failed_model) + + if fb_client is not None: fb_resp = _call_fallback_candidate_sync( fb_client, fb_model, fb_label, task=task, messages=messages, @@ -8217,11 +9143,21 @@ def call_llm( reasoning_config=reasoning_config) if fb_resp is not None: return fb_resp - # The candidate was quarantined (stale credentials or an - # unavailable deployment). Restart selection from the first - # configured layer; unhealthy entries are skipped, so the - # next viable candidate is selected in declared order. - fallback_reason = "stale fallback credential" + # The candidate had a stale/unrefreshable credential and was + # quarantined — walk the discovery chain once more; unhealthy + # entries are skipped so the next viable candidate serves. + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason="stale fallback credential") + if fb_client is not None: + fb_resp = _call_fallback_candidate_sync( + fb_client, fb_model, fb_label, + task=task, messages=messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, effective_timeout=effective_timeout, + effective_extra_body=effective_extra_body, + reasoning_config=reasoning_config) + if fb_resp is not None: + return fb_resp # All fallback layers exhausted — emit a single user-visible # warning so the operator knows aux task is about to fail. # (#26882) The error itself is re-raised below. @@ -8300,6 +9236,7 @@ def extract_content_or_reasoning(response) -> str: return "" +@_relay_auxiliary_call_async async def async_call_llm( task: str = None, *, @@ -8391,6 +9328,11 @@ async def async_call_llm( f"Run: hermes setup") effective_timeout = _effective_aux_timeout(task, timeout) + _set_relay_auxiliary_route( + resolved_provider, + final_model, + resolved_api_mode, + ) # Pass the client's actual base_url (not just resolved_base_url) so # endpoint-specific temperature overrides can distinguish @@ -8408,12 +9350,35 @@ async def async_call_llm( kwargs["messages"] = _convert_openai_images_to_anthropic(kwargs["messages"]) try: - # Retry on the same provider for a transient transport blip before the - # except-chain escalates to fallback — see call_llm() for the rationale. - # The async path honors the same configured retry budget as sync. + # Retry ONCE on the same provider for a transient transport blip + # before the except-chain escalates to fallback — see call_llm() + # for the rationale. (PR #16587) + _force_stream_async = ( + _provider_requires_stream( + resolved_provider, _client_base or resolved_base_url, + ) + and not isinstance(client, ( + AsyncCodexAuxiliaryClient, + AsyncAnthropicAuxiliaryClient, + AsyncBedrockAuxiliaryClient, + )) + ) + + async def _acreate(_kwargs: Dict[str, Any]) -> Any: + if _force_stream_async: + return await _acreate_with_stream(client, _kwargs, task) + return await client.chat.completions.create(**_kwargs) + try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task, + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task, provider=resolved_provider, base_url=_client_base) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -8428,29 +9393,20 @@ async def async_call_llm( transient_err, ) raise - max_transient_retries = _transient_retry_count() - last_transient = transient_err - for attempt in range(1, max_transient_retries + 1): - backoff = min( - _TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (attempt - 1)), - 8.0, - ) - logger.info( - "Auxiliary %s (async): transient transport error " - "(attempt %d/%d); retrying same provider after %.1fs " - "before fallback: %s", - task or "call", attempt, max_transient_retries, backoff, - last_transient, - ) - await asyncio.sleep(backoff) - try: - return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) - except Exception as retry_transient: - if not _is_transient_transport_error(retry_transient): - raise - last_transient = retry_transient - raise last_transient + logger.info( + "Auxiliary %s (async): transient transport error; retrying " + "once on the same provider before fallback: %s", + task or "call", transient_err, + ) + return _validate_llm_response( + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + create=_acreate, + ), + task) except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) @@ -8461,14 +9417,18 @@ async def async_call_llm( ) try: return _validate_llm_response( - await client.chat.completions.create(**retry_kwargs), task) + await _relay_async_completion( + client, + retry_kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: retry_err_str = str(retry_err) if not ( _is_payment_error(retry_err) or _is_connection_error(retry_err) or _is_auth_error(retry_err) - or _is_provider_deployment_unavailable_error(retry_err) or "max_tokens" in retry_err_str or "unsupported_parameter" in retry_err_str ): @@ -8496,16 +9456,16 @@ async def async_call_llm( kwargs.pop("max_completion_tokens", None) try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: # If the max_tokens retry also hits a payment or connection # error, fall through to the fallback chain below. - if not ( - _is_payment_error(retry_err) - or _is_connection_error(retry_err) - or _is_rate_limit_error(retry_err) - or _is_provider_deployment_unavailable_error(retry_err) - ): + if not (_is_payment_error(retry_err) or _is_connection_error(retry_err) or _is_rate_limit_error(retry_err)): raise first_err = retry_err @@ -8530,7 +9490,12 @@ async def async_call_llm( kwargs["model"] = healed_model try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: first_err = retry_err @@ -8562,7 +9527,12 @@ async def async_call_llm( kwargs["model"] = refreshed_model try: return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not ( _is_auth_error(retry_err) @@ -8589,7 +9559,12 @@ async def async_call_llm( if refreshed_model and refreshed_model != kwargs.get("model"): kwargs["model"] = refreshed_model return _validate_llm_response( - await refreshed_client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + refreshed_client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) # ── Auth refresh retry (mirrors sync call_llm) ─────────────── auth_refresh_provider = _auth_refresh_provider_for_route( @@ -8624,15 +9599,7 @@ async def async_call_llm( ) # ── Same-provider credential-pool recovery (mirrors sync) ───── - pool_provider = _recoverable_pool_provider( - resolved_provider, - client, - main_runtime=main_runtime, - route_is_explicit=_route_has_explicit_endpoint_or_key( - resolved_base_url, - resolved_api_key, - ), - ) + pool_provider = _recoverable_pool_provider(resolved_provider, client, main_runtime=main_runtime) _client_api_key = str(getattr(client, "api_key", "") or "") if pool_provider and (_is_auth_error(first_err) or _is_payment_error(first_err) or _is_rate_limit_error(first_err)): recovery_err = first_err @@ -8641,7 +9608,12 @@ async def async_call_llm( if _is_rate_limit_error(first_err) and not _is_payment_error(first_err): try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _relay_async_completion( + client, + kwargs, + provider=resolved_provider, + api_mode=resolved_api_mode, + ), task) except Exception as retry_err: if not (_is_auth_error(retry_err) or _is_payment_error(retry_err) or _is_rate_limit_error(retry_err)): raise @@ -8687,7 +9659,6 @@ async def async_call_llm( or _is_connection_error(first_err) or _is_rate_limit_error(first_err) or _is_model_incompatible_error(first_err) - or _is_provider_deployment_unavailable_error(first_err) or _is_invalid_aux_response_error(first_err) ) # Capacity errors (payment/quota/connection/rate-limit) bypass the @@ -8703,7 +9674,6 @@ async def async_call_llm( or _is_connection_error(first_err) or _is_rate_limit_error(first_err) or _is_model_incompatible_error(first_err) - or _is_provider_deployment_unavailable_error(first_err) or _is_invalid_aux_response_error(first_err) ) if should_fallback and (is_auto or is_capacity_error): @@ -8711,32 +9681,13 @@ async def async_call_llm( reason = "auth error" elif _is_payment_error(first_err): reason = "payment error" - _mark_recoverable_provider_unhealthy( - resolved_provider, - client, - main_runtime=main_runtime, - task=task, - route_is_explicit=_route_has_explicit_endpoint_or_key( - resolved_base_url, - resolved_api_key, - ), + _mark_provider_unhealthy( + _recoverable_pool_provider(resolved_provider, client) or resolved_provider ) elif _is_rate_limit_error(first_err): reason = "rate limit" elif _is_model_incompatible_error(first_err): reason = "model incompatible with route" - elif _is_provider_deployment_unavailable_error(first_err): - reason = "provider deployment unavailable" - _mark_recoverable_provider_unhealthy( - resolved_provider, - client, - main_runtime=main_runtime, - task=task, - route_is_explicit=_route_has_explicit_endpoint_or_key( - resolved_base_url, - resolved_api_key, - ), - ) elif _is_invalid_aux_response_error(first_err): reason = "invalid provider response" else: @@ -8744,42 +9695,42 @@ async def async_call_llm( logger.info("Auxiliary %s (async): %s on %s (%s), trying fallback", task or "call", reason, resolved_provider, first_err) + # Narrow the configured-chain skip to the exact model that + # failed ONLY for model-specific failures. Auth (401) and + # payment (402) errors are provider-wide — the credentials or + # account behind every model on that provider are the same — so + # a sibling model can't recover; keep skipping the whole + # provider so the main-agent-model safety net is still reached. + _chain_failed_model = ( + None if reason in ("auth error", "payment error") else final_model + ) # Fallback order (#26882, #26803): # 1. User-configured fallback_chain (per-task) if set # 2. For auto: top-level main fallback_providers/fallback_model # 3. For auto: built-in auxiliary discovery chain # 4. For explicit aux providers: main agent model safety net - attempted_fallbacks: set[tuple[str, Optional[str]]] = set() - fallback_reason = reason - while True: - fb_client, fb_model, fb_label = (None, None, "") - if is_auto: - fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=fallback_reason) - if fb_client is None: - fb_client, fb_model, fb_label = _try_main_fallback_chain( - task, resolved_provider or "auto", reason=fallback_reason) - if fb_client is None: - fb_client, fb_model, fb_label = _try_payment_fallback( - resolved_provider, task, reason=fallback_reason) - else: - fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=fallback_reason) - if fb_client is None: - fb_client, fb_model, fb_label = _try_main_agent_model_fallback( - resolved_provider, task, reason=fallback_reason) - + fb_client, fb_model, fb_label = (None, None, "") + if is_auto: + fb_client, fb_model, fb_label = _try_configured_fallback_chain( + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) if fb_client is None: - break - fallback_identity = (fb_label, fb_model) - if fallback_identity in attempted_fallbacks: - logger.warning( - "Auxiliary %s (async): fallback selector repeated %s; " - "stopping to avoid a retry loop", - task or "call", fb_label, - ) - break - attempted_fallbacks.add(fallback_identity) + fb_client, fb_model, fb_label = _try_main_fallback_chain( + task, resolved_provider or "auto", reason=reason) + if fb_client is None: + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason=reason) + else: + fb_client, fb_model, fb_label = _try_configured_fallback_chain( + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) + if fb_client is None: + fb_client, fb_model, fb_label = _try_main_agent_model_fallback( + resolved_provider, task, reason=reason, + failed_model=_chain_failed_model) + + if fb_client is not None: + # Convert sync fallback client to async async_fb, async_fb_model = _to_async_client( fb_client, fb_model or "", is_vision=(task == "vision") ) @@ -8792,7 +9743,23 @@ async def async_call_llm( reasoning_config=reasoning_config) if fb_resp is not None: return fb_resp - fallback_reason = "stale fallback credential" + # Stale/unrefreshable candidate credential — quarantined; walk + # the discovery chain once more (unhealthy entries skipped). + fb_client, fb_model, fb_label = _try_payment_fallback( + resolved_provider, task, reason="stale fallback credential") + if fb_client is not None: + async_fb, async_fb_model = _to_async_client( + fb_client, fb_model or "", is_vision=(task == "vision") + ) + fb_resp = await _call_fallback_candidate_async( + async_fb, async_fb_model or fb_model, fb_label, + task=task, messages=messages, + temperature=temperature, max_tokens=max_tokens, + tools=tools, effective_timeout=effective_timeout, + effective_extra_body=effective_extra_body, + reasoning_config=reasoning_config) + if fb_resp is not None: + return fb_resp # All fallback layers exhausted — warn before re-raising. (#26882) logger.warning( "Auxiliary %s (async): %s on %s and all fallbacks exhausted " diff --git a/agent/azure_identity_adapter.py b/agent/azure_identity_adapter.py index 9506715019d7..dd0f62ab9737 100644 --- a/agent/azure_identity_adapter.py +++ b/agent/azure_identity_adapter.py @@ -367,11 +367,27 @@ class name. Users wanting the precise class can run with info["tenant_id_env"] = os.environ["AZURE_TENANT_ID"].strip() # Surface which env-var sources are present without minting yet. + # Credential-bearing vars (AZURE_CLIENT_SECRET, AZURE_FEDERATED_TOKEN_FILE) + # are read through the profile secret scope so a multiplexed profile's + # diagnostics don't report another profile's env-bridged credentials; + # unscoped CLI probes keep the legacy env read (Slack pattern). + def _scoped_env(name: str) -> str: + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return os.environ.get(name, "").strip() + env_sources = [] - if os.environ.get("AZURE_FEDERATED_TOKEN_FILE", "").strip(): + if _scoped_env("AZURE_FEDERATED_TOKEN_FILE"): env_sources.append("WorkloadIdentityCredential (AZURE_FEDERATED_TOKEN_FILE)") if (os.environ.get("AZURE_CLIENT_ID", "").strip() - and os.environ.get("AZURE_CLIENT_SECRET", "").strip() + and _scoped_env("AZURE_CLIENT_SECRET") and os.environ.get("AZURE_TENANT_ID", "").strip()): env_sources.append("EnvironmentCredential (client secret)") if os.environ.get("IDENTITY_ENDPOINT", "").strip() or os.environ.get("MSI_ENDPOINT", "").strip(): diff --git a/agent/backend_identity.py b/agent/backend_identity.py new file mode 100644 index 000000000000..7a7e9efb6bfe --- /dev/null +++ b/agent/backend_identity.py @@ -0,0 +1,204 @@ +"""Single owner for backend identity and failure-scoped skip decisions. + +Every fallback / dedup / skip / quarantine decision in Hermes ultimately asks +one question: **"is this candidate the same backend as the one that failed, +along the axis that failure invalidated?"** Before this module, that +question was re-implemented inline at six call sites across four subsystems, +each comparing whatever string was locally convenient (provider label, +provider+model, base_url+model, ...). Each incident fixed one site while the +others kept the bug: #22548 (same-shim aliases), #70893 (xai-oauth vs xai — +same host, distinct credential), #59561 (aux chain skipped sibling models), +#72468 (aux main-model safety net, same bug three weeks later), #62984 / +#54250 / #57584 (dedup ignoring base_url strands multi-endpoint pools). + +The root insight: "provider" conflates three independent identity axes, and +each failure class invalidates a different one: + +* **credential surface** — auth 401 / payment 402 kill everything sharing the + credential (every model, every host reached with that key/token). +* **endpoint** — DNS failure / connection refused kill everything behind the + URL, regardless of model or credential. +* **model deployment** — timeout / overload / rate limit / model-incompatible + kill ONE model's deployment. A sibling model behind the same URL is an + independent deployment (real incident: aux ``glm-5.2`` hung and timed out + while main ``macaron-v1-venti`` on the identical endpoint was serving + 448K-token turns). + +Call sites should build :class:`BackendIdentity` values, classify the failure +with :func:`classify_failure_scope`, and ask :func:`should_skip_candidate`. +Do not re-implement any comparison inline — extend THIS module instead. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from enum import Enum +from typing import Optional + +logger = logging.getLogger(__name__) + + +class FailureScope(Enum): + """Which identity axis a failure invalidates.""" + + #: Timeout, overload/429, connection blip, model-incompatible, invalid + #: response: evidence against ONE model deployment only. + MODEL = "model" + #: Auth 401 / payment 402: evidence against the shared credential — + #: every model reached with it is equally dead. + CREDENTIAL = "credential" + #: DNS / connection-refused / unreachable host: evidence against the + #: endpoint — every model behind the URL is equally dead. + ENDPOINT = "endpoint" + + +#: Reason strings already used by auxiliary_client's except-chain, mapped to +#: scopes. Unknown reasons default to MODEL — the least-invalidating scope — +#: so an unrecognized failure never over-skips viable candidates. +_REASON_SCOPES = { + "auth error": FailureScope.CREDENTIAL, + "payment error": FailureScope.CREDENTIAL, + "rate limit": FailureScope.MODEL, + "model incompatible with route": FailureScope.MODEL, + "invalid provider response": FailureScope.MODEL, + "connection error": FailureScope.MODEL, + "timeout": FailureScope.MODEL, +} + + +def classify_failure_scope(reason: Optional[str]) -> FailureScope: + """Map a human-readable failure reason to the identity axis it kills.""" + return _REASON_SCOPES.get((reason or "").strip().lower(), FailureScope.MODEL) + + +def _norm_provider(value: Optional[str]) -> str: + return (value or "").strip().lower() + + +def _norm_model(value: Optional[str]) -> str: + return (value or "").strip().lower() + + +def _norm_base_url(value: Optional[str]) -> str: + return (value or "").strip().rstrip("/").lower() + + +@dataclass(frozen=True) +class BackendIdentity: + """Normalized identity of one (provider, model, endpoint) deployment. + + Empty fields mean "unknown" — comparisons treat an unknown axis as + non-distinguishing (it can neither prove sameness nor difference on its + own; the remaining axes decide). + """ + + provider: str = "" + model: str = "" + base_url: str = "" + + @classmethod + def build( + cls, + provider: Optional[str] = None, + model: Optional[str] = None, + base_url: Optional[str] = None, + ) -> "BackendIdentity": + return cls( + provider=_norm_provider(provider), + model=_norm_model(model), + base_url=_norm_base_url(base_url), + ) + + +def _both_first_class(a: BackendIdentity, b: BackendIdentity) -> bool: + """True when both providers are distinct registered first-class providers. + + Two different registry providers have distinct credential surfaces even + when they share an inference host (xai-oauth vs xai, openai-codex vs + openai-api) — #70893. Custom/shim aliases are NOT in the registry, so + two aliases pointing at one URL still count as the same backend (#22548). + """ + if not a.provider or not b.provider or a.provider == b.provider: + return False + try: + from hermes_cli.auth import PROVIDER_REGISTRY + + return a.provider in PROVIDER_REGISTRY and b.provider in PROVIDER_REGISTRY + except Exception: + return False + + +def same_credential_surface(a: BackendIdentity, b: BackendIdentity) -> bool: + """Do two identities share the credential a 401/402 just invalidated? + + Conservative on purpose: an unprovable axis must answer "different" + (try the candidate — worst case one wasted RTT) rather than "same" + (skip — worst case stranded failover). Two distinct custom labels at + one URL may carry different per-entry api_keys, so a shared URL alone + never proves a shared credential; it is only used as a weak signal + when a provider label is missing entirely. + """ + if a.provider and b.provider: + # Same label = same configured credential. Different labels = + # different credential config (first-class registry providers + # explicitly so — #70893; custom entries can each carry their own + # api_key, so sameness is unprovable and we must not skip). + return a.provider == b.provider + # Provider unknown on a side: same explicit URL is the best signal left. + return bool(a.base_url and a.base_url == b.base_url) + + +def same_endpoint(a: BackendIdentity, b: BackendIdentity) -> bool: + """Do two identities sit behind the endpoint that just went unreachable?""" + if a.base_url and b.base_url: + return a.base_url == b.base_url + # An unknown base_url inherits the provider default → same provider + # label implies the same default endpoint. + return bool(a.provider and a.provider == b.provider) + + +def same_deployment(a: BackendIdentity, b: BackendIdentity) -> bool: + """Are these the exact same model deployment (the thing a timeout kills)? + + Provider+model must match; the base_url axis distinguishes only when BOTH + sides carry an explicit URL (#62984: same provider+model on two different + explicit URLs is two deployments — a pool). A side with an unknown URL + inherits the provider default and cannot prove difference. + """ + if not (a.provider and b.provider and a.provider == b.provider): + # Same-host different-label shims: same URL + same model IS the same + # deployment even when the alias labels differ (#22548) — unless both + # labels are first-class registry providers (#70893). + if ( + a.base_url + and a.base_url == b.base_url + and a.model + and a.model == b.model + and not _both_first_class(a, b) + ): + return True + return False + if not (a.model and b.model and a.model == b.model): + return False + if a.base_url and b.base_url and a.base_url != b.base_url: + return False # distinct explicit endpoints — a pool, not a dup + return True + + +def should_skip_candidate( + candidate: BackendIdentity, + failed: BackendIdentity, + scope: FailureScope = FailureScope.MODEL, +) -> bool: + """THE skip predicate: would trying ``candidate`` just repeat the failure? + + True when the candidate is the same backend as ``failed`` along the axis + ``scope`` says the failure invalidated. Every fallback/dedup/skip site + must call this instead of comparing labels inline. + """ + if scope is FailureScope.CREDENTIAL: + return same_credential_surface(candidate, failed) + if scope is FailureScope.ENDPOINT: + return same_endpoint(candidate, failed) + return same_deployment(candidate, failed) diff --git a/agent/background_review.py b/agent/background_review.py index c2ea87bd94e2..01820cfffdf2 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -18,6 +18,7 @@ from __future__ import annotations +import copy import json import logging import os @@ -70,8 +71,8 @@ def _resolve_review_runtime(agent: Any) -> Dict[str, Any]: "routed": False, } try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() except Exception: return parent aux = cfg.get("auxiliary", {}) if isinstance(cfg.get("auxiliary"), dict) else {} @@ -209,7 +210,10 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] "conversation for skills the user loaded via /skill-name or you " "read via skill_view. If any of them covers the territory of the " "new learning, PATCH that one first. It is the skill that was in " - "play, so it's the right one to extend.\n" + "play, so it's the right one to extend — but only if it is " + "curator-managed. Bundled, hub, pinned, and user-owned skills are " + "off-limits to you no matter how relevant (see Protected skills " + "below); for those, fall through to the next option.\n" " 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). " "If no loaded skill fits but an existing class-level skill does, " "patch it. Add a subsection, a pitfall, or broaden a trigger.\n" @@ -251,10 +255,18 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] "Protected skills (DO NOT edit these):\n" " • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n" " • Hub-installed skills (installed via 'hermes skills install').\n" - "Pinned skills (marked via 'hermes curator pin') CAN be improved — " - "pin only blocks deletion/archive/consolidation by the curator, not " - "content updates. Patch them when a pitfall or missing step turns up, " - "same as any other agent-created skill.\n" + " • Skills in skills.external_dirs (externally owned).\n" + " • PINNED skills (marked via 'hermes curator pin'). You are an " + "autonomous no-user-present actor, so pin blocks your writes too — " + "content updates included. Only the user, in a foreground session, " + "can change a pinned skill.\n" + " • USER-OWNED skills — anything not curator-managed. A skill the " + "user hand-wrote, installed by URL, or asked a foreground agent to " + "create is theirs, not yours; your writes to it WILL be refused. " + "This includes skills that were loaded or consulted this session: " + "being in play does not make one yours to edit. If such a skill is " + "wrong or outdated, say so in your reply and recommend " + "'hermes curator adopt ' — do not try to patch it.\n" "If the only skills that need updating are protected, say\n" "'Nothing to save.' and stop.\n\n" "Do NOT capture (these become persistent self-imposed constraints " @@ -273,6 +285,15 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] " • One-off task narratives. A user asking 'summarize today's " "market' or 'analyze this PR' is not a class of work that warrants " "a skill.\n\n" + " • Unresolved failures: if the session ended WITHOUT actually " + "finding a working method — you tried several things, none worked, " + "and told the user to check manually — do NOT write those attempts " + "up as a 'reliable workflow' or 'recommended approach'. That presents " + "an untested sequence of failures as validated guidance a future " + "session will trust and repeat. Either say 'Nothing to save', or, " + "only if you are independently confident of a real working alternative " + "(not something you are merely guessing might work), capture ONLY that " + "alternative — never the dead ends, and never dressed up as best practice.\n\n" "If a tool failed because of setup state, capture the FIX (install " "command, config step, env var to set) under an existing setup or " "troubleshooting skill — never 'this tool does not work' as a " @@ -309,7 +330,9 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] " 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were " "loaded via /skill-name or skill_view in the conversation. If one " "of them covers the learning, PATCH it first. It was in play; " - "it's the right place.\n" + "it's the right place — provided it is curator-managed. Protected " + "and user-owned skills are off-limits however relevant; fall " + "through when one of those is the best fit.\n" " 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to " "find the right one). Patch it.\n" " 3. ADD A SUPPORT FILE under an existing umbrella via " @@ -337,10 +360,15 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] "Protected skills (DO NOT edit these):\n" " • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n" " • Hub-installed skills (installed via 'hermes skills install').\n" - "Pinned skills (marked via 'hermes curator pin') CAN be improved — " - "pin only blocks deletion/archive/consolidation by the curator, not " - "content updates. Patch them when a pitfall or missing step turns up, " - "same as any other agent-created skill.\n" + " • Skills in skills.external_dirs (externally owned).\n" + " • PINNED skills (marked via 'hermes curator pin'). Pin blocks " + "autonomous writes entirely — content updates included — because no " + "user is present to consent. Only a foreground session can change one.\n" + " • USER-OWNED skills — anything not curator-managed (hand-written, " + "URL-installed, or created by a foreground agent at the user's " + "request). Your writes to these WILL be refused, including to skills " + "loaded or consulted this session. If one is wrong, say so in your " + "reply and recommend 'hermes curator adopt ' instead.\n" "If the only skills that need updating are protected, say\n" "'Nothing to save.' and stop.\n\n" "Do NOT capture as skills (these become persistent self-imposed " @@ -359,6 +387,15 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] " • One-off task narratives. A user asking 'summarize today's " "market' or 'analyze this PR' is not a class of work that warrants " "a skill.\n\n" + " • Unresolved failures: if the session ended WITHOUT actually " + "finding a working method — you tried several things, none worked, " + "and told the user to check manually — do NOT write those attempts " + "up as a 'reliable workflow' or 'recommended approach'. That presents " + "an untested sequence of failures as validated guidance a future " + "session will trust and repeat. Either say 'Nothing to save', or, " + "only if you are independently confident of a real working alternative " + "(not something you are merely guessing might work), capture ONLY that " + "alternative — never the dead ends, and never dressed up as best practice.\n\n" "If a tool failed because of setup state, capture the FIX (install " "command, config step, env var to set) under an existing setup or " "troubleshooting skill — never 'this tool does not work' as a " @@ -709,6 +746,43 @@ def _bg_review_auto_deny(command, description, **kwargs): # _cached_system_prompt below. if not _routed: _fork_kwargs["reasoning_config"] = getattr(agent, "reasoning_config", None) + # Gateway session context is appended to the parent's cached + # system prompt at API-call time through this field. Preserve + # it on same-model forks so the complete effective system + # prompt remains byte-identical and can reuse the warm prefix. + _fork_kwargs["ephemeral_system_prompt"] = getattr( + agent, "ephemeral_system_prompt", None + ) + # Prefill messages are inserted immediately after the system + # message at API-call time (chat_completion_helpers.py / + # conversation_loop.py), so a parent with prefill configured + # (gateway prefill_messages_file) would otherwise diverge + # from the warm prefix at message index 1 — same bug class + # as the ephemeral prompt above, one position later. + # Deep copy: the unicode-error recovery path mutates + # prefill entries IN PLACE (_sanitize_messages_surrogates + # via conversation_loop), so sharing dicts would let a + # fork-side sanitize rewrite the parent's prefill bytes. + _parent_prefill = copy.deepcopy( + getattr(agent, "prefill_messages", None) or [] + ) + if _parent_prefill: + _fork_kwargs["prefill_messages"] = _parent_prefill + # OpenRouter provider-routing pins: prompt caches live per + # UPSTREAM provider, so a fork without the parent's pins can + # be routed to a different upstream and miss the warm cache + # even with byte-identical prompt/tools bytes. + for _pref_attr in ( + "providers_allowed", + "providers_ignored", + "providers_order", + "provider_sort", + "provider_require_parameters", + "provider_data_collection", + ): + _pref_val = getattr(agent, _pref_attr, None) + if _pref_val: + _fork_kwargs[_pref_attr] = _pref_val review_agent = AIAgent( model=_rt.get("model") or agent.model, max_iterations=16, diff --git a/agent/billing_usage.py b/agent/billing_usage.py index 2ac762bc2b31..c3ab6203e30c 100644 --- a/agent/billing_usage.py +++ b/agent/billing_usage.py @@ -34,7 +34,7 @@ import logging import math import os -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, Optional logger = logging.getLogger(__name__) diff --git a/agent/billing_view.py b/agent/billing_view.py index a535aee1f142..01a804e65f09 100644 --- a/agent/billing_view.py +++ b/agent/billing_view.py @@ -17,7 +17,7 @@ import logging import os import uuid -from dataclasses import dataclass, field +from dataclasses import dataclass from decimal import Decimal, InvalidOperation from typing import Any, Optional @@ -107,6 +107,22 @@ def display(self) -> str: return f"{self.masked} — {label}" if label else self.masked +@dataclass(frozen=True) +class PaymentMethodInfo: + """The payment method on file. `kind` is "card", "link", or "unknown" + — anything else is normalised to "unknown" at parse time, so consumers + only ever see fields that belong to the kind they are looking at.""" + + kind: str + brand: Optional[str] = None + last4: Optional[str] = None + wallet: Optional[str] = None + email: Optional[str] = None + resolved_via: Optional[str] = None + #: What the server called it, when we did not recognise the kind. + raw_kind: Optional[str] = None + + @dataclass(frozen=True) class MonthlyCap: limit_usd: Optional[Decimal] = None @@ -150,6 +166,7 @@ class BillingState: min_usd: Optional[Decimal] = None max_usd: Optional[Decimal] = None card: Optional[CardInfo] = None + payment_method: Optional[PaymentMethodInfo] = None monthly_cap: Optional[MonthlyCap] = None auto_reload: Optional[AutoReload] = None portal_url: Optional[str] = None @@ -201,6 +218,41 @@ def _parse_card(raw: Any) -> Optional[CardInfo]: return CardInfo(brand=brand, last4=last4, resolved_via=resolved_via) +def _parse_payment_method(raw: Any) -> Optional[PaymentMethodInfo]: + if not isinstance(raw, dict): + return None + kind = raw.get("kind") + if not isinstance(kind, str): + return None + + def _optional_string(key: str) -> Optional[str]: + value = raw.get(key) + return value if isinstance(value, str) else None + + resolved_via = _optional_string("resolvedVia") + brand = _optional_string("brand") + last4 = _optional_string("last4") + # Settle the kind here, the way _parse_card settles a card, so nothing + # downstream has to re-check which fields this kind is allowed to have. + if kind == "card" and brand and last4: + return PaymentMethodInfo( + kind="card", + brand=brand, + last4=last4, + wallet=_optional_string("wallet"), + resolved_via=resolved_via, + ) + if kind == "link": + return PaymentMethodInfo( + kind="link", + email=_optional_string("email"), + resolved_via=resolved_via, + ) + return PaymentMethodInfo( + kind="unknown", raw_kind=kind, resolved_via=resolved_via + ) + + def _parse_monthly_cap(raw: Any) -> Optional[MonthlyCap]: if not isinstance(raw, dict): return None @@ -274,6 +326,7 @@ def billing_state_from_payload( min_usd=parse_money(bounds.get("minUsd")), max_usd=parse_money(bounds.get("maxUsd")), card=_parse_card(payload.get("card")), + payment_method=_parse_payment_method(payload.get("paymentMethod")), monthly_cap=_parse_monthly_cap(payload.get("monthlyCap")), auto_reload=_parse_auto_reload(payload.get("autoReload")), portal_url=portal_url, diff --git a/agent/browser_provider.py b/agent/browser_provider.py index 75e88e584f31..3e96fa9c85e6 100644 --- a/agent/browser_provider.py +++ b/agent/browser_provider.py @@ -26,6 +26,7 @@ "session_name": str, # unique name for agent-browser --session "bb_session_id": str, # provider session ID (for close/cleanup) "cdp_url": str, # CDP websocket URL + "expires_at": str, # optional provider-authoritative ISO timestamp "features": dict, # feature flags that were enabled "external_call_id": str, # optional, managed-gateway billing key } @@ -96,6 +97,7 @@ def create_session(self, task_id: str) -> Dict[str, object]: "session_name": str, # unique name for agent-browser --session "bb_session_id": str, # provider session ID (for close/cleanup) "cdp_url": str, # CDP websocket URL + "expires_at": str, # optional provider-authoritative ISO timestamp "features": dict, # feature flags that were enabled } diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 37113b61acf0..e6e8ab7fdc1f 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -15,6 +15,7 @@ from __future__ import annotations +import contextvars import json import logging import math @@ -57,6 +58,12 @@ _FALLBACK_EXHAUSTED_COOLDOWN_S = 5.0 +def _context_thread_target(callback): + """Bind a no-argument thread target to the caller's ContextVars.""" + context = contextvars.copy_context() + return lambda: context.run(callback) + + def _ra(): """Lazy ``run_agent`` reference. @@ -188,6 +195,31 @@ def _provider_preferences_for_agent(agent) -> Dict[str, Any]: return preferences +def _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs: dict) -> dict: + """Merge Portal ``tags`` / ``session_id`` onto an Anthropic Messages kwargs dict. + + The Nous provider profile is only consulted by the OpenAI-wire transport; + anthropic_messages callers must merge it themselves. Passes ``session_id`` + only — not ``provider_preferences`` (those become a top-level ``provider`` + routing object on the OpenAI wire). Never blocks a turn on tagging. + """ + if getattr(agent, "provider", None) not in {"nous", "nous-portal", "nousresearch"}: + return anthropic_kwargs + try: + from providers import get_provider_profile + + nous_profile = get_provider_profile("nous") + if nous_profile is not None: + anthropic_kwargs.setdefault("extra_body", {}).update( + nous_profile.build_extra_body( + session_id=getattr(agent, "session_id", None) + ) + ) + except Exception as exc: # noqa: BLE001 — never block a turn on tagging + logger.debug("Nous Portal extra_body merge failed: %s", exc) + return anthropic_kwargs + + def _env_float(name: str, default: float) -> float: try: return float(os.getenv(name, str(default))) @@ -195,6 +227,53 @@ def _env_float(name: str, default: float) -> float: return default +def _estimate_chunk_bytes(chunk: Any) -> int: + """Cheap per-chunk size estimate for the stream diagnostic counters. + + The previous implementation used ``len(repr(chunk))`` — a full recursive + repr of a pydantic model on EVERY streaming chunk (5.5-8.8 µs each, + ~20-30 ms of pure CPU on a 3,000-chunk response, in the hottest loop in + the agent). The counter only feeds a retry-diagnostic log line, so an + estimate based on the delta payload lengths is plenty (2.1-2.4 µs, ~3x + cheaper, and independent of model/pydantic field count). Chat Completions + chunks are sized from their delta content/reasoning/tool-argument strings + plus a small framing constant; anything shape-unknown (Anthropic events, + stub providers) falls back to a flat constant so `bytes` stays monotonic + and roughly proportional to traffic. + """ + size = 40 # SSE/JSON framing floor per chunk + try: + choices = getattr(chunk, "choices", None) + if choices: + delta = getattr(choices[0], "delta", None) + if delta is not None: + for attr in ("content", "reasoning_content", "reasoning"): + v = getattr(delta, attr, None) + if isinstance(v, str): + size += len(v) + tool_calls = getattr(delta, "tool_calls", None) + if tool_calls: + for tc in tool_calls: + fn = getattr(tc, "function", None) + if fn is not None: + args = getattr(fn, "arguments", None) + if isinstance(args, str): + size += len(args) + name = getattr(fn, "name", None) + if isinstance(name, str): + size += len(name) + else: + # Non-chat-completions shapes (Anthropic events etc.): try the + # common text fields, else keep the framing floor. + for attr in ("text", "partial_json"): + v = getattr(getattr(chunk, "delta", None), attr, None) + if isinstance(v, str): + size += len(v) + except Exception: + pass + return size + + def _codex_wait_notice_recovery( *, stale_timeout: float, @@ -433,26 +512,56 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): def should_use_direct_api_call(agent) -> bool: - """Whether a cron OpenAI-wire request should skip the interrupt worker. - - Issue #62151 is specific to OpenRouter's chat-completions path inside the - gateway cron thread stack. Keep native/Codex/Bedrock/MoA transports on their - established workers: their cancellation and client ownership differ, and - the report provides no evidence that those paths share the pre-HTTP wedge. + """Whether an OpenAI-wire request should skip the interrupt worker. + + Two nested-pool contexts wedge before the socket opens when the request + is pushed onto yet another daemon worker thread: + + - Gateway cron turns (#62151): gateway asyncio loop → cron thread → + interrupt worker. Fixed by running inline. + - Delegated children (#60203): gateway loop → async-delegation executor + (module-lifetime daemon pool) → per-child timeout executor → interrupt + worker. Same fingerprint after multi-day gateway uptime — children hang + at their FIRST API call with zero stale-detector output (the worker + never reaches dispatch), all providers, restart cures it. The cron fix + originally excluded delegation "for lack of evidence"; #60203 is that + evidence. + + Running inline drops the deepest thread layer (whose only job is + interactive-interrupt responsiveness). Interrupts still work: the inline + path registers ``agent._active_request_abort``, which ``interrupt()`` + invokes cross-thread to shut the active sockets — the same mechanism the + async-delegation stall monitor (#72227) relies on. + + Keep native/Codex/Bedrock/MoA transports on their established workers: + their cancellation and client ownership differ. """ - return ( - getattr(agent, "platform", None) == "cron" - and getattr(agent, "api_mode", None) == "chat_completions" - and getattr(agent, "provider", None) != "moa" - ) + if getattr(agent, "api_mode", None) != "chat_completions": + return False + if getattr(agent, "provider", None) == "moa": + return False + if getattr(agent, "platform", None) == "cron": + return True + # Delegated child (delegate_task sync or background) — detected via the + # execution ContextVar set by _run_single_child, with the agent's own + # platform stamp as a fallback for callers that bypass the runner. + try: + from agent.delegation_context import is_delegated_child_context + + if is_delegated_child_context(): + return True + except Exception: + pass + return getattr(agent, "platform", None) == "subagent" def direct_api_call(agent, api_kwargs: dict): """Run a non-streaming LLM call inline on the conversation thread. - Used when ``should_use_direct_api_call`` is True. Skips the interrupt worker - (whose only job is interactive-interrupt responsiveness, which this context - does not have) so the nested-pool deadlock (#62151) cannot occur. Because the + Used when ``should_use_direct_api_call`` is True (cron turns and + delegated children). Skips the interrupt worker (whose only job is + interactive-interrupt responsiveness, which these contexts do not have) + so the nested-pool deadlock (#62151, #60203) cannot occur. Because the request runs in-flight normally, the per-request OpenAI client's own httpx timeout (provider ``request_timeout_seconds`` / ``HERMES_API_TIMEOUT``) bounds a genuinely hung provider — the same bound interactive calls already rely on. @@ -463,11 +572,17 @@ def direct_api_call(agent, api_kwargs: dict): request_client_lock = threading.Lock() def _abort_active_request(reason: str) -> None: - """Abort the inline request from cron's watchdog/interrupt thread.""" + """Abort the inline request from a watchdog/interrupt thread.""" + # Abort while still holding the holder lock: the instant it is + # released, the inline finally may pop + cache the client for reuse + # and the NEXT call check it out — a late abort would then poison + # the slot and shut down an innocent in-flight request's sockets + # (same atomicity contract as _close_request_client_once in the + # interruptible variants; the abort itself never blocks). with request_client_lock: request_client = request_client_holder["client"] - if request_client is not None: - agent._abort_request_openai_client(request_client, reason=reason) + if request_client is not None: + agent._abort_request_openai_client(request_client, reason=reason) def _make_client(reason: str, kind: str = "openai"): # direct_api_call only runs for OpenAI-wire chat_completions cron @@ -480,6 +595,10 @@ def _make_client(reason: str, kind: str = "openai"): agent._active_request_abort = _abort_active_request return client + # Only a clean return may report the reuse reason (request_complete): + # after an error or interrupt the wire client is really closed so the + # retry builds a fresh pool (see _REQUEST_CLIENT_REUSE_REASONS). + succeeded = False try: response = _dispatch_nonstreaming_api_request( agent, api_kwargs, make_client=_make_client @@ -492,6 +611,7 @@ def _make_client(reason: str, kind: str = "openai"): if getattr(agent, "_interrupt_requested", False): raise InterruptedError("Agent interrupted during API call") _reset_stale_streak(agent) + succeeded = True return response finally: if getattr(agent, "_active_request_abort", None) is _abort_active_request: @@ -500,7 +620,10 @@ def _make_client(reason: str, kind: str = "openai"): request_client = request_client_holder["client"] request_client_holder["client"] = None if request_client is not None: - agent._close_request_openai_client(request_client, reason="request_complete") + agent._close_request_openai_client( + request_client, + reason="request_complete" if succeeded else "request_error_cleanup", + ) def interruptible_api_call(agent, api_kwargs: dict): @@ -578,20 +701,28 @@ def _close_request_client_once(reason: str) -> None: and owner_tid is not None and owner_tid != threading.get_ident() ) - if not stranger_thread: - # Owning thread (or no recorded owner) → pop and fully close. - request_client_holder["client"] = None - request_client_holder["owner_tid"] = None + if stranger_thread: + # Abort while still holding the holder lock: the instant it + # is released, the worker's finally may pop + cache the client + # for reuse and the NEXT call check it out — an abort landing + # after that would poison the slot and shut down an innocent + # in-flight request's sockets. The abort itself never blocks + # (socket shutdown + slot poison), so holding the lock across + # it only delays the racing pop, never the data path. + if request_client_kind.get("value", "openai") == "anthropic_messages": + agent._abort_request_anthropic_client( + request_client, reason=reason + ) + else: + agent._abort_request_openai_client(request_client, reason=reason) + return + # Owning thread (or no recorded owner) → pop and fully close. + request_client_holder["client"] = None + request_client_holder["owner_tid"] = None if request_client is None: return - kind = request_client_kind.get("value", "openai") - if kind == "anthropic_messages": - if stranger_thread: - agent._abort_request_anthropic_client(request_client, reason=reason) - else: - agent._close_request_anthropic_client(request_client, reason=reason) - elif stranger_thread: - agent._abort_request_openai_client(request_client, reason=reason) + if request_client_kind.get("value", "openai") == "anthropic_messages": + agent._close_request_anthropic_client(request_client, reason=reason) else: agent._close_request_openai_client(request_client, reason=reason) @@ -628,7 +759,15 @@ def _call(): return result["error"] = e finally: - _close_request_client_once("request_complete") + # Reuse reason only on a clean response; any other outcome — + # error, or the cancel-swallow return above (which leaves both + # result slots None) — really closes so the next attempt builds + # a fresh pool (see _REQUEST_CLIENT_REUSE_REASONS). + _close_request_client_once( + "request_complete" + if result["response"] is not None + else "request_error_cleanup" + ) # ── Stale-call timeout (mirrors streaming stale detector) ──────── # Non-streaming calls return nothing until the full response is @@ -755,7 +894,7 @@ def _call(): _call_start = time.time() agent._touch_activity("waiting for non-streaming API response") - t = threading.Thread(target=_call, daemon=True) + t = threading.Thread(target=_context_thread_target(_call), daemon=True) t.start() _poll_count = 0 while t.is_alive(): @@ -981,9 +1120,10 @@ def _call(): -def build_api_kwargs(agent, api_messages: list) -> dict: +def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = None) -> dict: """Build the keyword arguments dict for the active API mode.""" - tools_for_api = agent.tools + if tools_for_api is None: + tools_for_api = agent.tools if agent.api_mode == "anthropic_messages": _transport = agent._get_transport() @@ -993,7 +1133,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict: ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None) if ephemeral_out is not None: agent._ephemeral_max_output_tokens = None # consume immediately - return _transport.build_kwargs( + anthropic_kwargs = _transport.build_kwargs( model=agent.model, messages=anthropic_messages, tools=tools_for_api, @@ -1006,6 +1146,12 @@ def build_api_kwargs(agent, api_messages: list) -> dict: fast_mode=(agent.request_overrides or {}).get("speed") == "fast", drop_context_1m_beta=bool(getattr(agent, "_oauth_1m_beta_disabled", False)), ) + # Nous Portal reads ``tags`` and ``session_id`` as top-level body fields + # on its Messages route the same way it does on /chat/completions, but + # the profile hook that produces them is only consulted by the + # OpenAI-wire transport. Merge them here so Messages traffic keeps + # product attribution and sticky routing. + return _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs) # AWS Bedrock native Converse API — bypasses the OpenAI client entirely. # The adapter handles message/tool conversion and boto3 calls directly. @@ -1076,6 +1222,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict: tools=tools_for_api, reasoning_config=agent.reasoning_config, session_id=getattr(agent, "session_id", None), + base_url=agent.base_url, max_tokens=agent.max_tokens, timeout=agent._resolved_api_call_timeout(), request_overrides=agent.request_overrides, @@ -1311,6 +1458,17 @@ def build_assistant_message(agent, assistant_message, finish_reason: str) -> dic from agent.redact import redact_sensitive_text _san_content = redact_sensitive_text(_san_content) + # NOTE (empty-content class fix): textless assistant turns are NOT padded + # here. The single owner for "never send a turn strict wire validation + # rejects as empty" is ``repair_empty_non_final_messages`` in + # agent_runtime_helpers, which runs inside ``sanitize_api_messages`` — the + # unconditional pre-send chokepoint for both the main loop and the summary + # path. Padding at write time was tried (a single-space pad, later a + # placeholder) and rejected: it forked the concept across three sites, + # broke codex commentary turns (content:'' is a designed state there), and + # a DB-side pad can't survive ``_rows_to_conversation``'s whitespace strip + # anyway. Repair belongs at the send boundary, once. + msg = { "role": "assistant", "content": _san_content, @@ -1598,29 +1756,28 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool ) return agent._try_activate_fallback(reason) - # Skip entries that resolve to the current (provider, model) — falling - # back to the same backend that just failed loops the failure. Compare - # base_url too so two distinct custom_providers entries pointing at the - # same shim/proxy URL also dedup. See issue #22548. - current_provider = (getattr(agent, "provider", "") or "").strip().lower() - current_model = (getattr(agent, "model", "") or "").strip() - current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower() - fb_base_url_for_dedup = (fb.get("base_url") or "").strip().rstrip("/").lower() - if fb_provider == current_provider and fb_model == current_model: - logger.warning( - "Fallback skip: chain entry %s/%s matches current provider/model", - fb_provider, fb_model, - ) - return agent._try_activate_fallback(reason) - if ( - fb_base_url_for_dedup - and current_base_url - and fb_base_url_for_dedup == current_base_url - and fb_model == current_model - ): + # Skip entries that resolve to the same backend that just failed — + # falling back to it loops the failure. Identity semantics (which axes + # distinguish two backends, shim aliases, first-class credential + # surfaces, multi-endpoint pools) are owned by agent.backend_identity — + # see #22548, #70893, #62984. Do not re-implement comparisons here. + from agent.backend_identity import BackendIdentity, should_skip_candidate + + current_ident = BackendIdentity.build( + provider=getattr(agent, "provider", ""), + model=getattr(agent, "model", ""), + base_url=str(getattr(agent, "base_url", "") or ""), + ) + fb_ident = BackendIdentity.build( + provider=fb_provider, + model=fb_model, + base_url=(fb.get("base_url") or ""), + ) + if should_skip_candidate(fb_ident, current_ident): logger.warning( - "Fallback skip: chain entry base_url %s matches current backend", - fb_base_url_for_dedup, + "Fallback skip: chain entry %s/%s resolves to the same backend " + "as the current one (%s)", + fb_provider, fb_model, current_ident.base_url or current_ident.provider, ) return agent._try_activate_fallback(reason) @@ -1632,19 +1789,17 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool # Pass base_url and api_key from fallback config so custom # endpoints (e.g. Ollama Cloud) resolve correctly instead of # falling through to OpenRouter defaults. + from hermes_cli.fallback_config import resolve_entry_api_key + fb_base_url_hint = (fb.get("base_url") or "").strip() or None - fb_api_key_hint = (fb.get("api_key") or "").strip() or None - if not fb_api_key_hint: - # key_env and api_key_env are both documented aliases (see - # _normalize_custom_provider_entry in hermes_cli/config.py). - fb_key_env = (fb.get("key_env") or fb.get("api_key_env") or "").strip() - if fb_key_env: - fb_api_key_hint = os.getenv(fb_key_env, "").strip() or None + fb_api_key_hint = resolve_entry_api_key(fb) # For Ollama Cloud endpoints, pull OLLAMA_API_KEY from env # when no explicit key is in the fallback config. Host match # (not substring) — see GHSA-76xc-57q6-vm5m. if fb_base_url_hint and base_url_host_matches(fb_base_url_hint, "ollama.com") and not fb_api_key_hint: - fb_api_key_hint = os.getenv("OLLAMA_API_KEY") or None + from agent.secret_scope import get_secret + + fb_api_key_hint = get_secret("OLLAMA_API_KEY") or None fb_client, _resolved_fb_model = resolve_provider_client( fb_provider, model=fb_model, raw_codex=True, explicit_base_url=fb_base_url_hint, @@ -1671,6 +1826,14 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool _fb_is_azure = agent._is_azure_openai_url(fb_base_url) if fb_provider == "openai-codex": fb_api_mode = "codex_responses" + elif fb_provider in {"nous", "nous-portal", "nousresearch"}: + # Portal is dual-wire: anthropic/* must land on /v1/messages. + # resolve_provider_client still returns an OpenAI client for + # Nous; the anthropic_messages branch below rebuilds the native + # client from that credential + base_url. + from hermes_cli.providers import nous_api_mode + + fb_api_mode = nous_api_mode(fb_model) elif ( fb_provider == "anthropic" or fb_base_url.rstrip("/").lower().endswith("/anthropic") @@ -1739,6 +1902,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_provider, fb_model, _pool_provider, ) agent._credential_pool = None + agent._credential_pool_entry_id = None if getattr(agent, "_credential_pool", None) is None: try: from agent.credential_pool import load_pool @@ -1801,6 +1965,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool # not only after a later credential-rotation rebuild. agent._replace_primary_openai_client(reason="fallback_timeout_apply") + from agent.agent_runtime_helpers import sync_credential_pool_entry_id + sync_credential_pool_entry_id(agent) + # Re-evaluate prompt caching for the new provider/model agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy( @@ -1905,6 +2072,28 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: """Request a summary when max iterations are reached. Returns the final response text.""" print(f"⚠️ Reached maximum iterations ({agent.max_iterations}). Requesting summary...") + summary_api_request_id = f"iteration-summary:{uuid.uuid4()}" + summary_call_outcome = "failed" + + def _managed_summary_call(request, callback, *, retry_count: int): + from agent import relay_llm + + return relay_llm.execute_current( + request, + callback, + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + metadata={ + "api_mode": str( + getattr(agent, "api_mode", "") or "chat_completions" + ), + "api_request_id": summary_api_request_id, + "call_role": "iteration_summary", + "retry_count": retry_count, + }, + defer_logical_completion=True, + ) + summary_request = ( "You've reached the maximum number of tool-calling iterations allowed. " "Please provide a final response summarizing what you've found and accomplished so far, " @@ -2090,15 +2279,33 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if agent.api_mode == "anthropic_messages": _tsum = agent._get_transport() - _ant_kw = _tsum.build_kwargs(model=agent.model, messages=api_messages, tools=None, - max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - is_oauth=agent._is_anthropic_oauth, - preserve_dots=agent._anthropic_preserve_dots()) - summary_response = agent._anthropic_messages_create(_ant_kw) + _ant_kw = _tsum.build_kwargs( + model=agent.model, + messages=api_messages, + tools=None, + max_tokens=agent.max_tokens, + reasoning_config=agent.reasoning_config, + is_oauth=agent._is_anthropic_oauth, + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None), + ) + _ant_kw = _merge_nous_portal_messages_extra_body(agent, _ant_kw) + summary_response = _managed_summary_call( + _ant_kw, + agent._anthropic_messages_create, + retry_count=0, + ) _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_summary_result.content or "").strip() else: - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=0, + ) _summary_result = agent._get_transport().normalize_response(summary_response) final_response = (_summary_result.content or "").strip() @@ -2106,6 +2313,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if "" in final_response: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: + summary_call_outcome = "success" messages.append({"role": "assistant", "content": final_response}) else: final_response = "I reached the iteration limit and couldn't generate a summary." @@ -2120,11 +2328,22 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: final_response = (_cnr_retry.content or "").strip() elif agent.api_mode == "anthropic_messages": _tretry = agent._get_transport() - _ant_kw2 = _tretry.build_kwargs(model=agent.model, messages=api_messages, tools=None, - is_oauth=agent._is_anthropic_oauth, - max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - preserve_dots=agent._anthropic_preserve_dots()) - retry_response = agent._anthropic_messages_create(_ant_kw2) + _ant_kw2 = _tretry.build_kwargs( + model=agent.model, + messages=api_messages, + tools=None, + is_oauth=agent._is_anthropic_oauth, + max_tokens=agent.max_tokens, + reasoning_config=agent.reasoning_config, + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None), + ) + _ant_kw2 = _merge_nous_portal_messages_extra_body(agent, _ant_kw2) + retry_response = _managed_summary_call( + _ant_kw2, + agent._anthropic_messages_create, + retry_count=1, + ) _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_retry_result.content or "").strip() else: @@ -2141,7 +2360,14 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if summary_extra_body: summary_kwargs["extra_body"] = summary_extra_body - summary_response = agent._ensure_primary_openai_client(reason="iteration_limit_summary_retry").chat.completions.create(**summary_kwargs) + summary_client = agent._ensure_primary_openai_client( + reason="iteration_limit_summary_retry" + ) + summary_response = _managed_summary_call( + summary_kwargs, + lambda request: summary_client.chat.completions.create(**request), + retry_count=1, + ) _retry_result = agent._get_transport().normalize_response(summary_response) final_response = (_retry_result.content or "").strip() @@ -2149,6 +2375,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: if "" in final_response: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: + summary_call_outcome = "success" messages.append({"role": "assistant", "content": final_response}) else: final_response = "I reached the iteration limit and couldn't generate a summary." @@ -2156,8 +2383,15 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: final_response = "I reached the iteration limit and couldn't generate a summary." except Exception as e: - logger.warning(f"Failed to get summary response: {e}") + logger.warning("Failed to get summary response: %s", e) final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" + finally: + from agent import relay_llm + + relay_llm.complete_logical_call( + summary_api_request_id, + outcome=summary_call_outcome, + ) return final_response @@ -2190,7 +2424,7 @@ def cleanup_task_resources(agent, task_id: str) -> None: _ra().cleanup_vm(task_id) except Exception as e: if agent.verbose_logging: - logger.warning(f"Failed to cleanup VM for task {task_id}: {e}") + logger.warning("Failed to cleanup VM for task %s: %s", task_id, e) try: headed = False try: @@ -2208,7 +2442,7 @@ def cleanup_task_resources(agent, task_id: str) -> None: _ra().cleanup_browser(task_id) except Exception as e: if agent.verbose_logging: - logger.warning(f"Failed to cleanup browser for task {task_id}: {e}") + logger.warning("Failed to cleanup browser for task %s: %s", task_id, e) def _build_partial_stream_stub( @@ -2316,7 +2550,9 @@ def _fire_first(): pass def _bedrock_call(): + stream = None try: + from agent import relay_llm from agent.bedrock_adapter import ( _get_bedrock_runtime_client, invalidate_runtime_client, @@ -2325,44 +2561,40 @@ def _bedrock_call(): normalize_converse_response, stream_converse_with_callbacks, ) - region = api_kwargs.pop("__bedrock_region__", "us-east-1") - api_kwargs.pop("__bedrock_converse__", None) - client = _get_bedrock_runtime_client(region) - try: - raw_response = client.converse_stream(**api_kwargs) - except Exception as _bedrock_exc: - # IAM policies scoped to bedrock:InvokeModel only (no - # InvokeModelWithResponseStream) reject converse_stream() - # with AccessDeniedException. That denial is permanent for - # the session — fall back to the non-streaming converse() - # inline (it maps to bedrock:InvokeModel) and disable - # streaming for subsequent calls so we don't re-fail every - # turn. - if is_streaming_access_denied_error(_bedrock_exc): - agent._disable_streaming = True - agent._safe_print( - "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — " - "falling back to non-streaming InvokeModel.\n" - " Grant that action to restore streaming output.\n" - ) - logger.info( - "bedrock: converse_stream denied by IAM (%s) — " - "using non-streaming converse() for this session.", - type(_bedrock_exc).__name__, - ) - result["response"] = normalize_converse_response( - client.converse(**api_kwargs) - ) - return - # Evict the cached client on stale-connection failures - # so the outer retry loop builds a fresh client/pool. - if is_stale_connection_error(_bedrock_exc): - invalidate_runtime_client(region) - raise - - # Claim the delta sink for this bedrock stream (#65991) so a - # superseded attempt's callbacks are fenced by the sink guard. - claim_stream_writer(agent) + intercepted_events = [] + writer_token = {"value": None} + + def _open_bedrock_stream(next_api_kwargs: dict[str, Any]): + final_kwargs = dict(next_api_kwargs) + region = final_kwargs.pop("__bedrock_region__", "us-east-1") + final_kwargs.pop("__bedrock_converse__", None) + client = _get_bedrock_runtime_client(region) + try: + raw_response = client.converse_stream(**final_kwargs) + except Exception as _bedrock_exc: + # InvokeModel-only policies cannot open a stream. Keep + # the fallback inside the same managed Relay attempt so + # the real provider request and terminal response still + # share one lifecycle boundary. + if is_streaming_access_denied_error(_bedrock_exc): + agent._disable_streaming = True + agent._safe_print( + "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — " + "falling back to non-streaming InvokeModel.\n" + " Grant that action to restore streaming output.\n" + ) + logger.info( + "bedrock: converse_stream denied by IAM (%s) — " + "using non-streaming converse() for this session.", + type(_bedrock_exc).__name__, + ) + return normalize_converse_response( + client.converse(**final_kwargs) + ) + if is_stale_connection_error(_bedrock_exc): + invalidate_runtime_client(region) + raise + return raw_response.get("stream", []) def _on_text(text): _fire_first() @@ -2377,18 +2609,65 @@ def _on_reasoning(text): _fire_first() agent._fire_reasoning_delta(text) - result["response"] = stream_converse_with_callbacks( - raw_response, + def _finalize_bedrock_stream(): + return stream_converse_with_callbacks( + {"stream": list(intercepted_events)} + ) + + def _bedrock_stream_created(_stream: Any) -> None: + writer_token["value"] = claim_stream_writer(agent) + + def _accept_bedrock_event(_event: Any) -> bool: + token = writer_token["value"] + return token is None or stream_writer_is_current(agent, token) + + stream = relay_llm.stream( + dict(api_kwargs), + _open_bedrock_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "bedrock"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=_finalize_bedrock_stream, + on_stream_created=_bedrock_stream_created, + on_chunk=intercepted_events.append, + chunk_adapter=lambda chunk: chunk, + accept_chunk=_accept_bedrock_event, + completed_response_predicate=lambda response: bool( + getattr(response, "choices", None) + ), + metadata={ + "api_mode": "custom", + "api_request_id": getattr( + agent, "_current_api_request_id", None + ), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + streamed_response = stream_converse_with_callbacks( + {"stream": stream}, on_text_delta=_on_text if agent._has_stream_consumers() else None, on_tool_start=_on_tool, on_reasoning_delta=_on_reasoning if agent.reasoning_callback or agent.stream_delta_callback else None, on_interrupt_check=lambda: agent._interrupt_requested, on_event=lambda: _bedrock_last_event.__setitem__("t", time.time()), ) + result["response"] = stream.final_response or streamed_response except Exception as e: result["error"] = e + finally: + if stream is not None: + stream.close() - t = threading.Thread(target=_bedrock_call, daemon=True) + t = threading.Thread( + target=_context_thread_target(_bedrock_call), daemon=True + ) t.start() while t.is_alive(): t.join(timeout=0.3) @@ -2554,25 +2833,33 @@ def _close_request_client_once(reason: str) -> None: and owner_tid is not None and owner_tid != threading.get_ident() ) - if not stranger_thread: - request_client_holder["client"] = None - request_client_holder["owner_tid"] = None + if stranger_thread: + # Abort under the holder lock — see the non-streaming variant + # for why the holder read and the abort must be atomic (a late + # abort would otherwise hit the NEXT request's checkout). + if request_client_kind.get("value", "openai") == "anthropic_messages": + agent._abort_request_anthropic_client( + request_client, reason=reason + ) + else: + agent._abort_request_openai_client(request_client, reason=reason) + return + request_client_holder["client"] = None + request_client_holder["owner_tid"] = None if request_client is None: return + # Stranger threads returned under the lock above, so only the owner + # (or an any-thread-safe stream handle) reaches the close dispatch. if request_kind == "stream": _close_request_stream_handle(request_client, reason) elif request_kind == "anthropic_messages": - if stranger_thread: - agent._abort_request_anthropic_client(request_client, reason=reason) - else: - agent._close_request_anthropic_client(request_client, reason=reason) - elif stranger_thread: - agent._abort_request_openai_client(request_client, reason=reason) + agent._close_request_anthropic_client(request_client, reason=reason) else: agent._close_request_openai_client(request_client, reason=reason) first_delta_fired = {"done": False} deltas_were_sent = {"yes": False} # Track if any deltas were fired (for fallback) + provider_tool_in_flight = {"yes": False} # Wall-clock timestamp of the last real streaming chunk. The outer # poll loop uses this to detect stale connections that keep receiving # SSE keep-alive pings but no actual data. @@ -2592,11 +2879,29 @@ def _close_request_client_once(reason: str) -> None: "discarded_chunks": 0, "discarded_bytes": 0, } + managed_stream_holder = {"stream": None} + + def _set_managed_stream(stream: Any) -> Any: + managed_stream_holder["stream"] = stream + return stream + + def _close_managed_stream() -> None: + stream = managed_stream_holder.pop("stream", None) + if stream is None: + return + close = getattr(stream, "close", None) + if callable(close): + try: + close() + except Exception: + logger.debug("Managed provider stream cleanup failed", exc_info=True) def _start_stream_attempt() -> int: with stream_attempt_lock: stream_attempt_state["current"] += 1 - return int(stream_attempt_state["current"]) + attempt_id = int(stream_attempt_state["current"]) + provider_tool_in_flight["yes"] = False + return attempt_id def _cancel_current_stream_attempt(reason: str) -> None: with stream_attempt_lock: @@ -2706,107 +3011,6 @@ def _call_chat_completions(stream_attempt_id: int): # Cap connect/pool at 60s even when provider timeout is higher. # connect/pool cover TCP handshake, not model inference. _conn_cap = min(_base_timeout, 60.0) if _provider_timeout_cfg is not None else 30.0 - stream_kwargs = { - **api_kwargs, - "stream": True, - "timeout": _httpx.Timeout( - connect=_conn_cap, - read=_stream_read_timeout, - write=_base_timeout, - pool=_conn_cap, - ), - } - # OpenAI's `stream_options={"include_usage": True}` drives usage - # accounting on OpenAI-compatible endpoints (incl. the Gemini OpenAI - # compat shim and aggregators like OpenRouter). Google's *native* - # Gemini REST endpoint rejects the keyword outright - # (`Completions.create() got an unexpected keyword argument - # 'stream_options'`), so omit it only for that endpoint. - if not is_native_gemini_base_url(agent.base_url): - stream_kwargs["stream_options"] = {"include_usage": True} - request_client = _set_request_client( - agent._create_request_openai_client( - reason="chat_completion_stream_request", - api_kwargs=stream_kwargs, - ) - ) - # Reset stale-stream timer so the detector measures from this - # attempt's start, not a previous attempt's last chunk. - last_chunk_time["t"] = time.time() - agent._touch_activity("waiting for provider response (streaming)") - # Initialize per-attempt stream diagnostics so the retry block can - # reach for them after the stream dies. Lives on - # ``request_client_holder["diag"]`` for closure access. - _diag = agent._stream_diag_init() - request_client_holder["diag"] = _diag - stream = request_client.chat.completions.create(**stream_kwargs) - if agent.provider == "moa": - # The MoA facade is a shared singleton — abort/close of the - # registered client is a no-op, so register the stream handle - # itself for interrupt teardown (#57354). - stream = _set_request_stream_handle(stream) - # Claim the delta sink for THIS attempt (#65991). If a prior attempt's - # stream is somehow still alive (a stale-stream reconnect whose socket - # abort raced), this claim supersedes it so its late chunks are fenced - # out of the turn instead of interleaving with ours. - _writer_token = claim_stream_writer(agent) - - # Some OpenAI-compatible adapters (for example copilot-acp, and the MoA - # openai-codex aggregator) accept stream=True but still return a - # completed response object rather than an iterator of chunks. Treat - # that as "streaming unsupported" for the rest of this session instead - # of crashing on ``for chunk in stream`` with ``'types.SimpleNamespace' - # object is not iterable`` (#11732, #55933). - # - # Discriminate on the mere PRESENCE of a ``choices`` attribute, not on - # it being a non-empty list: an adapter may hand back a completed - # response whose ``choices`` is ``None`` or empty (an error / - # content-filter / terminal frame), and every such shape is still a - # whole response — not a token stream — that would crash iteration just - # the same. A genuine provider stream (SDK ``Stream`` object, - # generator) exposes no ``choices`` attribute, so it is left untouched. - if hasattr(stream, "choices"): - logger.info( - "Streaming request returned a final response object instead of " - "an iterator; switching %s/%s to non-streaming for this session.", - agent.provider or "unknown", - agent.model or "unknown", - ) - agent._disable_streaming = True - # An empty/None ``choices`` carries no message to surface; return the - # completed object as-is so the outer loop's normal invalid-response - # validation (conversation_loop.py) handles it via the retry path, - # never ``for chunk in stream``. - choices = stream.choices - first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None - message = getattr(first_choice, "message", None) - if message is not None: - reasoning_text = ( - getattr(message, "reasoning_content", None) - or getattr(message, "reasoning", None) - ) - if isinstance(reasoning_text, str) and reasoning_text: - _fire_first_delta() - agent._fire_reasoning_delta(reasoning_text) - content = getattr(message, "content", None) - if isinstance(content, str) and content: - _fire_first_delta() - agent._fire_stream_delta(content) - return stream - - # Capture rate limit headers from the initial HTTP response. - # The OpenAI SDK Stream object exposes the underlying httpx - # response via .response before any chunks are consumed. - agent._capture_rate_limits(getattr(stream, "response", None)) - agent._capture_credits(getattr(stream, "response", None)) - # Snapshot diagnostic headers (cf-ray, x-openrouter-provider, etc.) - # so they survive even when the stream dies before any chunk - # arrives. Best-effort; never raises. - agent._stream_diag_capture_response(_diag, getattr(stream, "response", None)) - - # Log OpenRouter response cache status when present. - agent._check_openrouter_cache_status(getattr(stream, "response", None)) - content_parts: list = [] tool_calls_acc: dict = {} tool_gen_notified: set = set() @@ -2821,19 +3025,125 @@ def _call_chat_completions(stream_attempt_id: int): role = "assistant" reasoning_parts: list = [] usage_obj = None - for chunk in stream: - # Stop the moment a newer attempt has claimed the delta sink - # (#65991): this attempt has been superseded, so it must neither - # fire deltas (incl. the tool-suppressed raw-callback path below) - # nor keep consuming a stream that would interleave into the turn. - if not stream_writer_is_current(agent, _writer_token): + _diag = agent._stream_diag_init() + request_client_holder["diag"] = _diag + _writer_token = {"value": None} + attempt_request_client = {"value": None} + + def _open_stream(next_api_kwargs: dict[str, Any]): + stream_kwargs = { + **next_api_kwargs, + "stream": True, + "timeout": _httpx.Timeout( + connect=_conn_cap, + read=_stream_read_timeout, + write=_base_timeout, + pool=_conn_cap, + ), + } + # Native Gemini rejects OpenAI's usage-streaming extension. + if not is_native_gemini_base_url(agent.base_url): + stream_kwargs["stream_options"] = {"include_usage": True} + request_client = _set_request_client( + agent._create_request_openai_client( + reason="chat_completion_stream_request", + api_kwargs=stream_kwargs, + ) + ) + attempt_request_client["value"] = request_client + last_chunk_time["t"] = time.time() + agent._touch_activity("waiting for provider response (streaming)") + return request_client.chat.completions.create(**stream_kwargs) + + def _stream_created(raw_stream: Any) -> None: + response = getattr(raw_stream, "response", None) + agent._capture_rate_limits(response) + agent._capture_credits(response) + agent._stream_diag_capture_response(_diag, response) + agent._check_openrouter_cache_status(response) + _writer_token["value"] = claim_stream_writer(agent) + + def _accept_stream_chunk(_chunk: Any) -> bool: + # A stale-attempt fence can win while Relay is handing an + # already-received tool-call chunk back to Hermes. Preserve only + # the fact that a tool call was in flight so retry policy does not + # misclassify the attempt as a partial text response. The chunk + # itself is still rejected below and never reaches callbacks. + try: + choices = getattr(_chunk, "choices", None) + delta = getattr(choices[0], "delta", None) if choices else None + if getattr(delta, "tool_calls", None): + provider_tool_in_flight["yes"] = True + except Exception: + pass + if not _stream_attempt_is_active(stream_attempt_id): + return False + token = _writer_token["value"] + if token is not None and not stream_writer_is_current(agent, token): logger.warning( "Streaming attempt superseded by a newer stream; stopping " "consumption to preserve the single-writer invariant " "(model=%s).", api_kwargs.get("model", "unknown"), ) - break + return False + # Record provider activity before Relay processes the chunk. This + # prevents the stale watchdog from cancelling a live stream while + # an interceptor or codec is still handling an already-received + # event. + last_chunk_time["t"] = time.time() + return True + + def _relay_final_response() -> dict[str, Any]: + tool_calls = [tool_calls_acc[index] for index in sorted(tool_calls_acc)] + return { + "model": model_name, + "choices": [ + { + "message": { + "role": role, + "content": "".join(content_parts) or None, + "reasoning_content": "".join(reasoning_parts) or None, + "tool_calls": tool_calls or None, + }, + "finish_reason": finish_reason or "stop", + } + ], + "usage": usage_obj, + } + + from agent import relay_llm + + stream = _set_managed_stream( + relay_llm.stream( + api_kwargs, + _open_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "provider"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=_relay_final_response, + on_stream_created=_stream_created, + accept_chunk=_accept_stream_chunk, + completed_response_predicate=lambda value: hasattr(value, "choices"), + metadata={ + "api_mode": "chat_completions", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + ) + if agent.provider == "moa": + # Hermes interrupts the managed stream; Relay retains sole + # ownership of closing the underlying provider stream. + _set_request_stream_handle(stream) + for chunk in stream: last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") @@ -2844,18 +3154,39 @@ def _call_chat_completions(stream_attempt_id: int): _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 if _diag.get("first_chunk_at") is None: _diag["first_chunk_at"] = last_chunk_time["t"] - # Approximate byte size from the chunk's repr — exact wire - # bytes aren't exposed by the SDK, but len(repr(chunk)) is - # a stable proxy for "how much content arrived" that - # survives stub provider differences. + # Approximate byte size from the chunk's delta payload — + # exact wire bytes aren't exposed by the SDK. A full + # repr() per chunk was 5.5-8.8 µs of pure CPU on the + # hottest loop in the agent; the delta-length estimate + # is ~3x cheaper and stays proportional to traffic. try: - _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(chunk)) + _diag["bytes"] = int(_diag.get("bytes", 0)) + _estimate_chunk_bytes(chunk) except Exception: pass except Exception: pass if agent._interrupt_requested: + # Abandoning a half-read SSE response leaves its connection + # permanently checked out of the httpx pool — and the partial + # response built below makes the worker's finally report a + # reuse-reason close, which would cache the client together + # with the leaked connection (each interrupt leaking one more + # until the pool exhausts). Close the stream here, on the + # owning thread, so the connection is released first. + try: + stream.close() + except Exception: + # Connection may still be checked out — poison the slot so + # the finally's close really closes the pool instead of + # caching it (owner-thread abort: shutdown is safe, and the + # FD release still happens in the finally below). + request_client = attempt_request_client["value"] + if request_client is not None: + agent._abort_request_openai_client( + request_client, + reason="interrupt_stream_close_failed", + ) break if not _stream_attempt_is_active(stream_attempt_id): @@ -2987,11 +3318,46 @@ def _call_chat_completions(stream_attempt_id: int): if hasattr(chunk, "usage") and chunk.usage: usage_obj = chunk.usage + _close_managed_stream() + if _stream_attempt_was_cancelled(stream_attempt_id): raise _httpx.RemoteProtocolError( f"stream attempt {stream_attempt_id} was superseded" ) + # Some OpenAI-compatible adapters accept ``stream=True`` but return a + # completed response. Relay records that attempt while Hermes preserves + # its existing switch-to-non-streaming behavior for later calls. + if stream.final_response is not None: + final_response = stream.final_response + logger.info( + "Streaming request returned a final response object instead of " + "an iterator; switching %s/%s to non-streaming for this session.", + agent.provider or "unknown", + agent.model or "unknown", + ) + agent._disable_streaming = True + choices = final_response.choices + first_choice = ( + choices[0] + if isinstance(choices, (list, tuple)) and choices + else None + ) + message = getattr(first_choice, "message", None) + if message is not None: + reasoning_text = ( + getattr(message, "reasoning_content", None) + or getattr(message, "reasoning", None) + ) + if isinstance(reasoning_text, str) and reasoning_text: + _fire_first_delta() + agent._fire_reasoning_delta(reasoning_text) + content = getattr(message, "content", None) + if isinstance(content, str) and content: + _fire_first_delta() + agent._fire_stream_delta(content) + return final_response + # Build mock response matching non-streaming shape full_content = "".join(content_parts) or None mock_tool_calls = None @@ -3153,72 +3519,95 @@ def _call_anthropic(request_client): # fabricated "successful" empty turn. saw_stream_event = False - # Reset stale-stream timer for this attempt last_chunk_time["t"] = time.time() - # Per-attempt diagnostic dict for the retry block to consume. _diag = agent._stream_diag_init() request_client_holder["diag"] = _diag - # Defensive: strip Responses-only kwargs (instructions, input, ...) - # that can leak in under an api_mode-flip race. The Anthropic SDK - # raises a non-retryable TypeError on them, killing the turn. See - # #31673 / sanitize_anthropic_kwargs(). + _writer_token = {"value": None} + _stream_context = {"manager": None, "stream": None} + base_final_message = None + + from agent import relay_llm from agent.anthropic_adapter import sanitize_anthropic_kwargs - sanitize_anthropic_kwargs( - api_kwargs, log_prefix=getattr(agent, "log_prefix", "") - ) - # Use the Anthropic SDK's streaming context manager - with request_client.messages.stream(**api_kwargs) as stream: + + accumulator = relay_llm.AnthropicStreamAccumulator() + + def _open_anthropic_stream(next_api_kwargs: dict[str, Any]): + final_kwargs = dict(next_api_kwargs) + sanitize_anthropic_kwargs( + final_kwargs, + log_prefix=getattr(agent, "log_prefix", ""), + ) + manager = request_client.messages.stream(**final_kwargs) + _stream_context["manager"] = manager + return manager.__enter__() + + def _anthropic_stream_created(raw_stream: Any) -> None: + _stream_context["stream"] = raw_stream # The Anthropic SDK exposes the raw httpx response on - # ``stream.response``. Snapshot diagnostic headers - # immediately so they survive a stream that dies before the - # first event. + # ``stream.response``. Snapshot diagnostics immediately so they + # survive a stream that dies before the first event. try: agent._stream_diag_capture_response( - _diag, getattr(stream, "response", None) + _diag, + getattr(raw_stream, "response", None), ) except Exception: pass - # Claim the delta sink for THIS attempt (#65991) — parity with the - # chat_completions path so a superseded anthropic stream is fenced. - _writer_token = claim_stream_writer(agent) + _writer_token["value"] = claim_stream_writer(agent) + + def _accept_anthropic_event(_event: Any) -> bool: + token = _writer_token["value"] + if token is None or stream_writer_is_current(agent, token): + return True + logger.warning( + "Anthropic streaming attempt superseded by a newer stream; " + "stopping consumption to preserve the single-writer " + "invariant (model=%s).", + api_kwargs.get("model", "unknown"), + ) + return False + + stream = _set_managed_stream( + relay_llm.stream( + api_kwargs, + _open_anthropic_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "anthropic"), + model_name=str(getattr(agent, "model", "") or ""), + finalizer=accumulator.finalize, + on_stream_created=_anthropic_stream_created, + on_chunk=accumulator.observe, + accept_chunk=_accept_anthropic_event, + metadata={ + "api_mode": "anthropic_messages", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + }, + defer_logical_completion=True, + ) + ) + try: for event in stream: - # Bail the instant a newer attempt supersedes this one so a - # stale stream can't interleave tokens into the turn. - if not stream_writer_is_current(agent, _writer_token): - logger.warning( - "Anthropic streaming attempt superseded by a newer " - "stream; stopping consumption to preserve the " - "single-writer invariant (model=%s).", - api_kwargs.get("model", "unknown"), - ) - break saw_stream_event = True - # Update stale-stream timer on every event so the - # outer poll loop knows data is flowing. Without - # this, the detector kills healthy long-running - # Opus streams after 180 s even when events are - # actively arriving (the chat_completions path - # already does this at the top of its chunk loop). last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") - - # Update per-attempt diagnostic counters (best-effort). try: _diag["chunks"] = int(_diag.get("chunks", 0)) + 1 if _diag.get("first_chunk_at") is None: _diag["first_chunk_at"] = last_chunk_time["t"] - try: - _diag["bytes"] = int(_diag.get("bytes", 0)) + len(repr(event)) - except Exception: - pass + _diag["bytes"] = int(_diag.get("bytes", 0)) + _estimate_chunk_bytes(event) except Exception: pass - if agent._interrupt_requested: break event_type = getattr(event, "type", None) - if event_type == "content_block_start": block = getattr(event, "content_block", None) if block and getattr(block, "type", None) == "tool_use": @@ -3227,7 +3616,6 @@ def _call_anthropic(request_client): if tool_name: _fire_first_delta() agent._fire_tool_gen_started(tool_name) - elif event_type == "content_block_delta": delta = getattr(event, "delta", None) if delta: @@ -3243,48 +3631,49 @@ def _call_anthropic(request_client): if thinking_text: _fire_first_delta() agent._fire_reasoning_delta(thinking_text) - - # Return the native Anthropic Message for downstream processing. - # If the stream was interrupted (the event loop broke out above on - # agent._interrupt_requested), do NOT call get_final_message() — on - # a partially-consumed stream the SDK may hang draining remaining - # events or return a Message with incomplete tool_use blocks (partial - # JSON in `input`). The outer poll loop raises InterruptedError, so - # this return value is discarded anyway. - if agent._interrupt_requested: - return None - # Zero-event guard (parity with the chat_completions zero-chunk - # guard above). Real SDK: an eventless stream has no - # message_start, so get_final_message() raises AssertionError - # (final-message snapshot is None) — normalize that to - # EmptyStreamError so it gets the transient retry budget - # instead of surfacing raw. + if not agent._interrupt_requested: + raw_stream = _stream_context["stream"] + if raw_stream is not None: + try: + base_final_message = raw_stream.get_final_message() + except AssertionError: + if not saw_stream_event: + raise EmptyStreamError( + "Provider returned an empty stream with no events " + "(possible upstream error or malformed event stream)." + ) from None + raise + finally: try: - _final_message = stream.get_final_message() - except AssertionError: - if not saw_stream_event: - raise EmptyStreamError( - "Provider returned an empty stream with no events " - "(possible upstream error or malformed event stream)." - ) from None - raise - # Shim variants of the same failure: an OpenAI-compat adapter - # may fabricate a contentless Message with no stop_reason, or - # return None where the SDK assert would have fired (e.g. - # ``python -O``). A real completed response always carries a - # stop_reason, so this cannot fire on legitimate turns. - if not saw_stream_event and ( - _final_message is None - or ( - not getattr(_final_message, "content", None) - and getattr(_final_message, "stop_reason", None) is None - ) - ): - raise EmptyStreamError( - "Provider returned an empty stream with no stop_reason " - "(possible upstream error or malformed event stream)." - ) - return _final_message + _close_managed_stream() + finally: + manager = _stream_context["manager"] + if manager is not None: + manager.__exit__(None, None, None) + + if agent._interrupt_requested: + return None + if ( + base_final_message is not None + and not getattr(base_final_message, "content", None) + and getattr(base_final_message, "stop_reason", None) is None + ): + raise EmptyStreamError( + "Provider returned an empty stream with no stop_reason " + "(possible upstream error or malformed event stream)." + ) + if base_final_message is not None and not stream.output_modified: + return base_final_message + final_message = accumulator.response(base_final_message) + if ( + not getattr(final_message, "content", None) + and getattr(final_message, "stop_reason", None) is None + ): + raise EmptyStreamError( + "Provider returned an empty stream with no stop_reason " + "(possible upstream error or malformed event stream)." + ) + return final_message def _call(): import httpx as _httpx @@ -3319,6 +3708,7 @@ def _call(): result["response"] = _call_chat_completions(stream_attempt_id) return # success except Exception as e: + _close_managed_stream() # If the main poll loop force-closed this request because # of an interrupt, the resulting transport error is the # expected consequence of our own close — NOT a transient @@ -3358,7 +3748,7 @@ def _call(): if deltas_were_sent["yes"]: _partial_tool_in_flight = bool( result.get("partial_tool_names") - ) + ) or provider_tool_in_flight["yes"] _is_sse_conn_err_preview = False if not _is_timeout and not _is_conn_err: from openai import APIError as _APIError @@ -3444,13 +3834,9 @@ def _call(): # already worker-owned-closed by _close_request_client_once # above; the next attempt builds a fresh one. The shared # _anthropic_client is never closed from inside a request. - if agent.api_mode != "anthropic_messages": - try: - agent._replace_primary_openai_client( - reason="stream_mid_tool_retry_pool_cleanup" - ) - except Exception: - pass + # #70773: same FD-recycle corruption vector for OpenAI. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next attempt. continue # SSE error events from proxies (e.g. OpenRouter sends @@ -3509,13 +3895,9 @@ def _call(): # above; next attempt builds fresh), so the shared # _anthropic_client is never closed from inside a # request — only the OpenAI-wire primary is refreshed. - if agent.api_mode != "anthropic_messages": - try: - agent._replace_primary_openai_client( - reason="stream_retry_pool_cleanup" - ) - except Exception: - pass + # #70773: same FD-recycle corruption vector for OpenAI. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next attempt. continue # Retries exhausted. Log the final failure with # full diagnostic detail (chain, headers, @@ -3596,7 +3978,7 @@ def _call(): " To avoid this delay, set display.streaming: false " "in config.yaml\n" ) - logger.info( + logger.exception( "Streaming failed before delivery: %s", e, ) @@ -3615,7 +3997,15 @@ def _call(): result["error"] = e return finally: - _close_request_client_once("stream_request_complete") + _close_managed_stream() + # Reuse reason only on a clean stream; any other outcome (error, + # cancel-swallow) really closes so the next attempt builds a + # fresh pool (see _REQUEST_CLIENT_REUSE_REASONS). + _close_request_client_once( + "stream_request_complete" + if result["response"] is not None + else "stream_error_cleanup" + ) # Provider-configured stale timeout takes priority over env default. _cfg_stale = get_provider_stale_timeout(agent.provider, agent.model) @@ -3637,9 +4027,9 @@ def _call(): # env var ``HERMES_LOCAL_STREAM_STALE_TIMEOUT`` overrides for escape-hatch. _local_default = 900.0 try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - _cfg = load_config() + _cfg = load_config_readonly() # read-only consumer — no deepcopy _agent_cfg = _cfg.get("agent") if isinstance(_cfg, dict) else None if isinstance(_agent_cfg, dict): _v = _agent_cfg.get("local_stream_stale_timeout") @@ -3677,7 +4067,7 @@ def _call(): if _reasoning_floor is not None: _stream_stale_timeout = max(_stream_stale_timeout, _reasoning_floor) - t = threading.Thread(target=_call, daemon=True) + t = threading.Thread(target=_context_thread_target(_call), daemon=True) t.start() _last_heartbeat = time.time() _HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches @@ -3758,10 +4148,15 @@ def _call(): # FD-recycle corruption vector. Nothing further is needed. pass else: - try: - agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup") - except Exception: - pass + # #70773: same FD-recycle corruption vector as #67142. + # The shared OpenAI client's connection pool must NOT be + # closed from this watchdog/poll thread — worker threads + # from previous stale-killed attempts may still be + # unwinding their SSL BIOs. The request-local client is + # already closed above via _close_request_client_once. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next request. + pass # Reset the timer so we don't kill repeatedly while # the inner thread processes the closure. last_chunk_time["t"] = time.time() @@ -3844,6 +4239,17 @@ def _call(): result["error"], ) _stub_finish_reason = FINISH_REASON_LENGTH + # NOTE (empty-content class fix): the stub is deliberately allowed + # to carry empty content here. The conversation loop's truncation + # path detects an EMPTY partial-stream stub (PARTIAL_STREAM_STUB_ID + # + no content) and skips appending it to history entirely — only + # the continuation nudge is sent. Substituting placeholder text at + # this site was tried and reverted: it defeats that guard (the stub + # no longer looks empty), gets appended to history, and the + # placeholder leaks into the stitched final response via + # truncated_response_parts. Transcripts that already carry a + # persisted empty turn are healed at the send boundary by + # ``repair_empty_non_final_messages`` (the single owner). _stub_msg = SimpleNamespace( role="assistant", content=_partial_text, tool_calls=None, reasoning_content=None, diff --git a/agent/codex_responses_adapter.py b/agent/codex_responses_adapter.py index bce372ebb5da..8f64f64b76f3 100644 --- a/agent/codex_responses_adapter.py +++ b/agent/codex_responses_adapter.py @@ -14,10 +14,12 @@ import json import logging import re +import unicodedata import uuid from types import SimpleNamespace from typing import Any, Dict, List, Optional +from agent.message_sanitization import deterministic_call_id from agent.prompt_builder import DEFAULT_AGENT_IDENTITY logger = logging.getLogger(__name__) @@ -72,6 +74,79 @@ def _classify_responses_issuer( ) +# The ChatGPT Codex backend reserves these Harmony wire tokens. If their +# literal spellings are replayed anywhere in request text, the backend rejects +# the request before inference with ``invalid_prompt: Request blocked.``. +# Category-Cf handling covers persisted sessions from an earlier U+200B weak +# defang; fullwidth bars survive format-character stripping while keeping the +# inspected source legible. +_HARMONY_CONTROL_TOKEN_RE = re.compile( + r"<\|(start|end|channel|message|constrain|return|call)\|>" +) +_FULLWIDTH_PIPE = "\uff5c" + + +def _neutralize_harmony_tokens(text: str) -> str: + """Keep Harmony source readable without emitting reserved wire tokens.""" + if not text or "<" not in text or "|" not in text: + return text + + replacement = rf"<{_FULLWIDTH_PIPE}\1{_FULLWIDTH_PIPE}>" + if not any(unicodedata.category(char) == "Cf" for char in text): + return _HARMONY_CONTROL_TOKEN_RE.sub(replacement, text) + + # U+200B is confirmed to be stripped by the Codex backend before its + # reserved-token check. Treat every Unicode format control equivalently so + # moving the character elsewhere in the token (or swapping in another Cf) + # cannot recreate the same visually hidden form. + visible_chars: List[str] = [] + original_positions: List[int] = [] + for index, char in enumerate(text): + if unicodedata.category(char) == "Cf": + continue + visible_chars.append(char) + original_positions.append(index) + + visible_text = "".join(visible_chars) + matches = list(_HARMONY_CONTROL_TOKEN_RE.finditer(visible_text)) + if not matches: + return text + + result: List[str] = [] + original_cursor = 0 + for match in matches: + original_start = original_positions[match.start()] + original_end = original_positions[match.end() - 1] + 1 + result.append(text[original_cursor:original_start]) + result.append(f"<{_FULLWIDTH_PIPE}{match.group(1)}{_FULLWIDTH_PIPE}>") + original_cursor = original_end + result.append(text[original_cursor:]) + return "".join(result) + + +def _neutralize_harmony_structure(value: Any) -> Any: + """Neutralize JSON-like values; normalize tuples and reject unsafe keys. + + Rewriting an object key could desynchronize a tool schema from the executor + contract, so a reserved token there is rejected explicitly instead. + """ + if isinstance(value, str): + return _neutralize_harmony_tokens(value) + if isinstance(value, (list, tuple)): + return [_neutralize_harmony_structure(item) for item in value] + if isinstance(value, dict): + normalized = {} + for key, item in value.items(): + if isinstance(key, str) and _neutralize_harmony_tokens(key) != key: + raise ValueError( + "Reserved Harmony tokens in a JSON object key cannot be " + "neutralized without changing its contract." + ) + normalized[key] = _neutralize_harmony_structure(item) + return normalized + return value + + # --------------------------------------------------------------------------- # Multimodal content helpers # --------------------------------------------------------------------------- @@ -182,12 +257,34 @@ def _summarize_user_message_for_log(content: Any, *, sep: str = " ") -> str: def _deterministic_call_id(fn_name: str, arguments: str, index: int = 0) -> str: """Generate a deterministic call_id from tool call content. - Used as a fallback when the API doesn't provide a call_id. + Thin wrapper over the single policy owner + ``agent.message_sanitization.deterministic_call_id`` (audit F4) — kept + as a module-level name because run_agent and tests import it from here. Deterministic IDs prevent cache invalidation — random UUIDs would make every API call's prefix unique, breaking OpenAI's prompt cache. """ - seed = f"{fn_name}:{arguments}:{index}" - digest = hashlib.sha256(seed.encode("utf-8", errors="replace")).hexdigest()[:12] + return deterministic_call_id(fn_name, arguments, index) + + +def _clamp_responses_call_id(call_id: str) -> str: + """Keep a ``call_id`` within the Responses API's 64-char limit (#73492). + + The codex app-server namespaces MCP tool call ids as + ``codex_mcp_____``; with an ``exec-`` + component the built-in ``hermes-tools`` server already overflows 64 chars, + and the Responses API rejects the whole payload with a non-retryable HTTP + 400 that then replays every turn — permanently bricking the session. + + Sibling defect to #10788 (which clamped ``input[*].id``), applied here to + ``call_id``. The surrogate is a pure, deterministic function of the + original, so the ``function_call`` and its matching ``function_call_output`` + — which carry the same original id — map to the same surrogate and stay + paired without correlating the two items. Short ids pass through unchanged, + preserving prompt-cache prefixes. + """ + if len(call_id) <= _MAX_RESPONSES_ITEM_ID_LENGTH: + return call_id + digest = hashlib.sha256(call_id.encode("utf-8", errors="replace")).hexdigest()[:32] return f"call_{digest}" @@ -546,7 +643,7 @@ def _chat_messages_to_responses_input( items.append({ "type": "function_call", - "call_id": call_id, + "call_id": _clamp_responses_call_id(call_id), "name": fn_name, "arguments": arguments, }) @@ -589,7 +686,7 @@ def _chat_messages_to_responses_input( items.append({ "type": "function_call_output", - "call_id": call_id, + "call_id": _clamp_responses_call_id(call_id), "output": output_value, }) @@ -604,10 +701,16 @@ def _preflight_codex_input_items( raw_items: Any, *, is_github_responses: bool = False, + sanitize_harmony_tokens: bool = False, ) -> List[Dict[str, Any]]: if not isinstance(raw_items, list): raise ValueError("Codex Responses input must be a list of input items.") + sanitize_text = ( + _neutralize_harmony_tokens + if sanitize_harmony_tokens + else lambda text: text + ) normalized: List[Dict[str, Any]] = [] seen_ids: set = set() for idx, item in enumerate(raw_items): @@ -628,7 +731,7 @@ def _preflight_codex_input_items( arguments = json.dumps(arguments, ensure_ascii=False) elif not isinstance(arguments, str): arguments = str(arguments) - arguments = arguments.strip() or "{}" + arguments = sanitize_text(arguments.strip() or "{}") normalized.append( { @@ -662,7 +765,7 @@ def _preflight_codex_input_items( if ptype == "input_text": text = part.get("text") if isinstance(text, str) and text: - cleaned.append({"type": "input_text", "text": text}) + cleaned.append({"type": "input_text", "text": sanitize_text(text)}) elif ptype == "input_image": url = part.get("image_url") if isinstance(url, str) and url: @@ -686,7 +789,7 @@ def _preflight_codex_input_items( { "type": "function_call_output", "call_id": call_id.strip(), - "output": output, + "output": sanitize_text(output), } ) continue @@ -699,14 +802,21 @@ def _preflight_codex_input_items( if item_id in seen_ids: continue seen_ids.add(item_id) - reasoning_item = {"type": "reasoning", "encrypted_content": encrypted} + reasoning_item: Dict[str, Any] = { + "type": "reasoning", + "encrypted_content": encrypted, + } # Do NOT include the "id" in the outgoing item — with # store=False (our default) the API tries to resolve the # id server-side and returns 404. The id is still used # above for local deduplication via seen_ids. summary = item.get("summary") if isinstance(summary, list): - reasoning_item["summary"] = summary + reasoning_item["summary"] = ( + _neutralize_harmony_structure(summary) + if sanitize_harmony_tokens + else summary + ) else: reasoning_item["summary"] = [] normalized.append(reasoning_item) @@ -735,7 +845,7 @@ def _preflight_codex_input_items( text = "" if not isinstance(text, str): text = str(text) - normalized_content.append({"type": "output_text", "text": text}) + normalized_content.append({"type": "output_text", "text": sanitize_text(text)}) if not normalized_content: raise ValueError(f"Codex Responses input[{idx}] message item must contain at least one text part.") normalized_item: Dict[str, Any] = { @@ -775,7 +885,7 @@ def _preflight_codex_input_items( for part_idx, part in enumerate(content): if isinstance(part, str): if part: - validated.append({"type": text_type, "text": part}) + validated.append({"type": text_type, "text": sanitize_text(part)}) continue if not isinstance(part, dict): raise ValueError( @@ -786,7 +896,7 @@ def _preflight_codex_input_items( text = part.get("text", "") if not isinstance(text, str): text = str(text or "") - validated.append({"type": text_type, "text": text}) + validated.append({"type": text_type, "text": sanitize_text(text)}) elif ptype in {"input_image", "image_url"}: image_ref = part.get("image_url", "") detail = part.get("detail") @@ -810,7 +920,7 @@ def _preflight_codex_input_items( if not isinstance(content, str): content = str(content) - normalized.append({"role": role, "content": content}) + normalized.append({"role": role, "content": sanitize_text(content)}) continue raise ValueError( @@ -825,6 +935,7 @@ def _preflight_codex_api_kwargs( *, allow_stream: bool = False, is_github_responses: bool = False, + sanitize_harmony_tokens: bool = False, ) -> Dict[str, Any]: if not isinstance(api_kwargs, dict): raise ValueError("Codex Responses request must be a dict.") @@ -845,10 +956,13 @@ def _preflight_codex_api_kwargs( if not isinstance(instructions, str): instructions = str(instructions) instructions = instructions.strip() or DEFAULT_AGENT_IDENTITY + if sanitize_harmony_tokens: + instructions = _neutralize_harmony_tokens(instructions) normalized_input = _preflight_codex_input_items( api_kwargs.get("input"), is_github_responses=is_github_responses, + sanitize_harmony_tokens=sanitize_harmony_tokens, ) tools = api_kwargs.get("tools") @@ -905,6 +1019,9 @@ def _preflight_codex_api_kwargs( } ) + if sanitize_harmony_tokens and normalized_tools is not None: + normalized_tools = _neutralize_harmony_structure(normalized_tools) + store = api_kwargs.get("store", False) if store is not False: raise ValueError("Codex Responses contract requires 'store' to be false.") @@ -912,7 +1029,8 @@ def _preflight_codex_api_kwargs( allowed_keys = { "model", "instructions", "input", "tools", "store", "reasoning", "include", "max_output_tokens", "temperature", - "tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier", + "tool_choice", "parallel_tool_calls", "prompt_cache_key", + "prompt_cache_retention", "service_tier", "extra_headers", "extra_body", "timeout", } normalized: Dict[str, Any] = { @@ -950,8 +1068,13 @@ def _preflight_codex_api_kwargs( if isinstance(temperature, (int, float)): normalized["temperature"] = float(temperature) - # Pass through tool_choice, parallel_tool_calls, prompt_cache_key - for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"): + # Pass through cache routing/retention and tool-dispatch hints. + for passthrough_key in ( + "tool_choice", + "parallel_tool_calls", + "prompt_cache_key", + "prompt_cache_retention", + ): val = api_kwargs.get(passthrough_key) if val is not None: normalized[passthrough_key] = val diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index 59f9bac25a26..c01084c4a449 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -18,7 +18,6 @@ import json import logging -import os import time from types import SimpleNamespace from typing import Any, Callable, Dict, List @@ -74,7 +73,10 @@ def _record_codex_app_server_usage(agent, turn) -> dict[str, Any]: try: if not agent._session_db_created: agent._ensure_db_session() - agent._session_db.update_token_counts( + # Enqueued for the SessionDB background writer — keeps the + # per-call accounting write off the turn thread (see + # conversation_loop's queue_token_counts call). + agent._session_db.queue_token_counts( agent.session_id, model=agent.model, billing_provider=agent.provider, @@ -154,7 +156,8 @@ def _record_codex_app_server_usage(agent, turn) -> dict[str, Any]: try: if not agent._session_db_created: agent._ensure_db_session() - agent._session_db.update_token_counts( + # Enqueued for the SessionDB background writer (see above). + agent._session_db.queue_token_counts( agent.session_id, input_tokens=canonical_usage.input_tokens, output_tokens=canonical_usage.output_tokens, @@ -778,12 +781,27 @@ def run_codex_app_server_turn( # the already-flushed user turn). See gateway/run.py agent_persisted. if getattr(agent, "_session_db", None) is not None: try: - agent._flush_messages_to_session_db(messages) + _codex_flush_ok = agent._flush_messages_to_session_db(messages) except Exception: - logger.debug( + _codex_flush_ok = False + logger.warning( "codex app-server projected-message flush failed", exc_info=True, ) + if _codex_flush_ok is False: + # Unlike the chat-completions loop (which fails closed BEFORE + # projection — see conversation_loop session_persistence_failed), + # codex output has already streamed to the user by the time this + # flush runs, so there is nothing left to withhold. We cannot + # flip agent_persisted=False either: the gateway fallback write + # would re-INSERT the already-flushed user turn (#860/#42039). + # Surface the durability gap loudly instead of a silent debug. + logger.warning( + "codex app-server turn was delivered but could NOT be " + "persisted to the session DB (session=%s) — this turn " + "will be missing after restart/resume", + getattr(agent, "session_id", None), + ) # Counter ticks for the agent-improvement loop. @@ -1221,6 +1239,8 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta """ import httpx as _httpx + from agent import relay_llm + active_client = client or agent._ensure_primary_openai_client(reason="codex_stream_direct") max_stream_retries = 1 # Accumulate streamed text so callers / compat shims can read it. @@ -1245,48 +1265,88 @@ def _on_event(event: Any) -> None: if agent._interrupt_requested: raise InterruptedError("Agent interrupted before Codex stream retry") - stream_kwargs = dict(api_kwargs) - stream_kwargs["stream"] = True + intercepted_events = [] + writer_token = {"value": None} + + def _open_codex_stream(next_api_kwargs: dict[str, Any]): + stream_kwargs = dict(next_api_kwargs) + stream_kwargs["stream"] = True + return active_client.responses.create(**stream_kwargs) + + def _codex_stream_created(_raw_stream: Any) -> None: + # Claim the delta sink for THIS physical attempt. A newer attempt + # supersedes this token and fences late deltas out of the turn. + writer_token["value"] = claim_stream_writer(agent) + + def _accept_codex_chunk(_chunk: Any) -> bool: + token = writer_token["value"] + if token is None or stream_writer_is_current(agent, token): + return True + logger.warning( + "Codex streaming attempt superseded by a newer stream; " + "stopping consumption to preserve the single-writer " + "invariant (model=%s).", + api_kwargs.get("model", "unknown"), + ) + return False + + def _finalize_codex_stream() -> Any: + return _consume_codex_event_stream( + list(intercepted_events), + model=api_kwargs.get("model"), + ) try: - event_stream = active_client.responses.create(**stream_kwargs) - except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: + event_stream = relay_llm.stream( + dict(api_kwargs), + _open_codex_stream, + session_id=str(getattr(agent, "session_id", "") or ""), + name=str(getattr(agent, "provider", "") or "codex"), + model_name=str(api_kwargs.get("model") or ""), + finalizer=_finalize_codex_stream, + on_stream_created=_codex_stream_created, + on_chunk=intercepted_events.append, + chunk_adapter=lambda chunk: chunk, + accept_chunk=_accept_codex_chunk, + completed_response_predicate=lambda response: bool( + hasattr(response, "output") and not hasattr(response, "__iter__") + ), + metadata={ + "api_mode": "codex_responses", + "api_request_id": getattr(agent, "_current_api_request_id", None), + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + "retry_count": attempt, + }, + defer_logical_completion=True, + ) + except ( + _httpx.RemoteProtocolError, + _httpx.ReadTimeout, + _httpx.ConnectError, + ConnectionError, + ) as exc: if attempt < max_stream_retries: logger.debug( - "Codex Responses stream connect failed (attempt %s/%s); retrying. %s error=%s", - attempt + 1, max_stream_retries + 1, - agent._client_log_context(), exc, + "Codex Responses stream connect failed (attempt %s/%s); " + "retrying. %s error=%s", + attempt + 1, + max_stream_retries + 1, + agent._client_log_context(), + exc, ) continue raise - # Claim the delta sink for THIS attempt (#65991) — parity with the - # chat_completions/anthropic/bedrock paths. If a prior attempt's - # stream is somehow still alive, this claim supersedes it so its - # late deltas are fenced out of the turn; conversely, a newer - # attempt supersedes us and the interrupt_check below stops our - # consumption immediately. - _writer_token = claim_stream_writer(agent) - - def _interrupt_or_superseded(_tok=_writer_token) -> bool: - if agent._interrupt_requested: - return True - if not stream_writer_is_current(agent, _tok): - logger.warning( - "Codex streaming attempt superseded by a newer stream; " - "stopping consumption to preserve the single-writer " - "invariant (model=%s).", - api_kwargs.get("model", "unknown"), - ) - return True - return False + def _interrupt_or_superseded() -> bool: + return bool(agent._interrupt_requested) try: - # Compatibility: some mocks/providers return a concrete response - # instead of an iterable. Pass it straight through. - if hasattr(event_stream, "output") and not hasattr(event_stream, "__iter__"): - return event_stream - try: final = _consume_codex_event_stream( event_stream, @@ -1315,6 +1375,29 @@ def _interrupt_or_superseded(_tok=_writer_token) -> bool: ) continue raise + except RuntimeError: + if event_stream.final_response is not None: + return event_stream.final_response + raise + + # A terminal response has already been assembled at this point + # (``final`` is built), so a transport error while draining the + # rest of the iterator — done only to let Relay run its response + # finalizer — must NOT discard it or trigger a new physical + # request. Record it as a non-fatal finalization warning and + # still return the already-completed, already-billed response. + if not agent._interrupt_requested: + try: + for _ignored in event_stream: + pass + except (_httpx.RemoteProtocolError, _httpx.ReadTimeout, _httpx.ConnectError, ConnectionError) as exc: + logger.warning( + "Codex Responses stream transport finalization failed " + "after a terminal response was already received; " + "returning the completed response instead of " + "retrying. %s error=%s", + agent._client_log_context(), exc, + ) if final.status in {"incomplete", "failed"}: logger.warning( @@ -1332,7 +1415,20 @@ def _interrupt_or_superseded(_tok=_writer_token) -> bool: try: close_fn() except Exception: - pass + # A failed close can leave this response's connection + # checked out of the httpx pool while the caller's finally + # reports a reuse-reason close (e.g. interrupt_check broke + # the event loop with collected output) — caching the + # client with the leaked connection. Poison the slot so + # that close really closes the pool (owner-thread abort; + # mirrors the chat-streaming interrupt-break handling). + # ``client is None`` means the shared primary client, + # which is never reuse-cached and must not have its + # sockets force-shut here. + if client is not None: + agent._abort_request_openai_client( + active_client, reason="codex_stream_close_failed" + ) def run_codex_create_stream_fallback(agent, api_kwargs: dict, client: Any = None): diff --git a/agent/coding_context.py b/agent/coding_context.py index 4a0cb8410308..aa38305e016f 100644 --- a/agent/coding_context.py +++ b/agent/coding_context.py @@ -337,9 +337,9 @@ def _coding_mode(config: Optional[dict[str, Any]]) -> str: """Return the normalized ``agent.coding_context`` mode (auto/focus/on/off).""" if config is None: try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - config = load_config() + config = load_config_readonly() except Exception: config = {} raw = ((config or {}).get("agent", {}) or {}).get("coding_context", "auto") @@ -520,30 +520,46 @@ def toolset_selection(self, config: Optional[dict[str, Any]] = None) -> Optional return None return [self.profile.toolset, *_enabled_mcp_servers(config)] - def system_blocks(self) -> list[str]: - """Stable system-prompt blocks for this posture (brief + workspace). + def system_prompt_parts(self) -> tuple[list[str], list[str], list[str]]: + """Return prefix, workspace, and trailing posture blocks separately. The operating brief carries a model-family edit-format nudge appended to it (one cached string, not a separate block) so the model is steered toward the `patch` mode it handles best — see ``_edit_format_line``. + + The three lists preserve the historical flat prompt order: the brief, + the live workspace snapshot, then configured operator instructions. + Prompt assembly can therefore put a cache boundary before the snapshot + without changing the persisted system-prompt bytes. """ if not self.is_coding: - return [] - blocks: list[str] = [] + return [], [], [] + prefix: list[str] = [] + workspace_parts: list[str] = [] + trailing: list[str] = [] if self.profile.guidance: brief = self.profile.guidance edit_line = _edit_format_line(self.model) if edit_line: brief = f"{brief}\n{edit_line}" - blocks.append(brief) + prefix.append(brief) workspace = build_coding_workspace_block(self.cwd) if workspace: - blocks.append(workspace) + workspace_parts.append(workspace) # Operator instructions ride their own block so the brief (block 0) stays # byte-stable and cache-keyed independently of user config. if self.instructions: - blocks.append(f"Operator instructions (from config):\n{self.instructions}") - return blocks + trailing.append(f"Operator instructions (from config):\n{self.instructions}") + return prefix, workspace_parts, trailing + + def system_blocks(self) -> list[str]: + """Return posture blocks in their historical display order. + + ``system_prompt_parts`` is the cache-aware API. This compatibility + helper retains the public flat list for callers outside prompt assembly. + """ + prefix, workspace, trailing = self.system_prompt_parts() + return [*prefix, *workspace, *trailing] def compact_skill_categories(self) -> frozenset[str]: """Skill categories to demote to names-only in the prompt's skill index. @@ -644,6 +660,19 @@ def coding_system_blocks( ).system_blocks() +def coding_system_prompt_parts( + *, + platform: Optional[str] = None, + cwd: Optional[str | Path] = None, + config: Optional[dict[str, Any]] = None, + model: Optional[str] = None, +) -> tuple[list[str], list[str], list[str]]: + """Return coding prefix, workspace snapshot, and trailing guidance.""" + return resolve_runtime_mode( + platform=platform, cwd=cwd, config=config, model=model + ).system_prompt_parts() + + def coding_compact_skill_categories( *, platform: Optional[str] = None, diff --git a/agent/context_breakdown.py b/agent/context_breakdown.py index 0e2eb772f2ff..4527c5dca220 100644 --- a/agent/context_breakdown.py +++ b/agent/context_breakdown.py @@ -154,3 +154,207 @@ def compute_session_context_breakdown( "estimated_total": estimated_total, "model": getattr(agent, "model", "") or "", } + + +# ── /context rendering (CLI + gateway) ────────────────────────────────────── +# +# Pure text renderers over the payload above. The CLI shows a glyph block-grid +# plus a category table; the gateway uses the same table without the grid +# (proportional monospace is not guaranteed on messaging platforms). + +_CATEGORY_GLYPHS = { + "system_prompt": "■", + "tool_definitions": "▣", + "rules": "▩", + "skills": "▤", + "mcp": "▥", + "subagent_definitions": "▦", + "memory": "▧", + "conversation": "▨", +} +_FREE_GLYPH = "·" +_GRID_COLUMNS = 20 +_GRID_ROWS = 5 # 100 cells → 1 cell per percent of the context window + +# Human-readable tables cap the expanded listings; nothing is dropped from +# the underlying data. +_DETAILS_TABLE_LIMIT = 15 + + +def _bytes_to_tokens(size: Optional[int]) -> Optional[int]: + if size is None: + return None + return (int(size) + 3) // 4 + + +def compute_context_details(agent: Any) -> Dict[str, Any]: + """Expanded per-skill / per-toolset cost listing for ``/context all``. + + Reuses the ``hermes prompt-size`` attribution mechanism (PR #66656): + per-skill index-line bytes parsed from the live ```` + block, and per-toolset schema bytes attributed via the tool registry's + canonical tool→toolset map. Byte figures are converted to the same + chars/4 token heuristic the categories above use. + """ + from hermes_cli.prompt_size import ( + _compute_skills_breakdown, + _compute_toolsets_breakdown, + ) + from agent.system_prompt import build_system_prompt_parts + + parts = build_system_prompt_parts(agent) + stable = parts.get("stable", "") or "" + skills_match = _SKILLS_BLOCK_RE.search(stable) + skills_block = skills_match.group(0) if skills_match else "" + + skills: List[Dict[str, Any]] = [] + if skills_block: + for entry in _compute_skills_breakdown(skills_block): + skills.append({ + "name": entry.get("name", ""), + "index_tokens": _bytes_to_tokens(entry.get("index_line_bytes")) or 0, + "skill_md_tokens": _bytes_to_tokens(entry.get("skill_md_bytes")), + }) + + toolsets: List[Dict[str, Any]] = [] + tools = list(getattr(agent, "tools", None) or []) + if tools: + for group in _compute_toolsets_breakdown(tools): + toolsets.append({ + "toolset": group.get("toolset", ""), + "tool_count": int(group.get("tool_count", 0) or 0), + "schema_tokens": _bytes_to_tokens(group.get("json_bytes")) or 0, + }) + + return {"skills": skills, "toolsets": toolsets} + + +def render_context_grid(payload: Dict[str, Any]) -> List[str]: + """Render the payload as a Claude Code-style glyph block grid. + + 100 cells (5×20), each one percent of the model context window. Categories + fill in declaration order; the remainder renders as free space. + """ + context_max = int(payload.get("context_max") or 0) + categories = payload.get("categories") or [] + total_cells = _GRID_COLUMNS * _GRID_ROWS + + cells: List[str] = [] + if context_max > 0: + for cat in categories: + tokens = int(cat.get("tokens") or 0) + n = round(tokens / context_max * total_cells) + if tokens > 0 and n == 0: + n = 1 # never render a nonzero category as invisible + glyph = _CATEGORY_GLYPHS.get(str(cat.get("id") or ""), "▪") + cells.extend([glyph] * n) + cells = cells[:total_cells] + cells.extend([_FREE_GLYPH] * (total_cells - len(cells))) + + return [ + " ".join(cells[row * _GRID_COLUMNS:(row + 1) * _GRID_COLUMNS]) + for row in range(_GRID_ROWS) + ] + + +def render_context_category_lines(payload: Dict[str, Any]) -> List[str]: + """Render the 'Estimated usage by category' table as plain-text lines.""" + categories = payload.get("categories") or [] + context_max = int(payload.get("context_max") or 0) + estimated_total = int(payload.get("estimated_total") or 0) + denom = context_max or estimated_total + + lines = ["Estimated usage by category"] + if not categories: + lines.append(" (no data yet — send a message first)") + return lines + + width = max(len(str(cat.get("label") or "")) for cat in categories) + width = max(width, len("Free space")) + for cat in categories: + tokens = int(cat.get("tokens") or 0) + glyph = _CATEGORY_GLYPHS.get(str(cat.get("id") or ""), "▪") + pct = tokens / denom * 100 if denom else 0.0 + label = str(cat.get("label") or cat.get("id") or "") + lines.append(f"{glyph} {label:<{width}} {tokens:>9,} tokens {pct:>5.1f}%") + if context_max > 0: + free = max(0, context_max - estimated_total) + pct = free / context_max * 100 + lines.append(f"{_FREE_GLYPH} {'Free space':<{width}} {free:>9,} tokens {pct:>5.1f}%") + return lines + + +def render_context_details_lines(details: Dict[str, Any]) -> List[str]: + """Render the expanded ``/context all`` per-skill / per-toolset tables.""" + lines: List[str] = [] + + toolsets = details.get("toolsets") or [] + if toolsets: + lines.append("Toolsets by schema cost (largest first)") + for group in toolsets[:_DETAILS_TABLE_LIMIT]: + lines.append( + f" {group['toolset']:<24} {group['tool_count']:>3} tools" + f" {group['schema_tokens']:>8,} tokens" + ) + remaining = len(toolsets) - _DETAILS_TABLE_LIMIT + if remaining > 0: + lines.append(f" … and {remaining} more") + + skills = details.get("skills") or [] + if skills: + if lines: + lines.append("") + lines.append("Skills by cost (index = always-on; SKILL.md = cost when loaded)") + for entry in skills[:_DETAILS_TABLE_LIMIT]: + name = str(entry.get("name") or "") + if len(name) > 28: + name = name[:27] + "…" + md = entry.get("skill_md_tokens") + md_str = f"{md:>8,}" if md is not None else f"{'n/a':>8}" + lines.append( + f" {name:<28} index {entry['index_tokens']:>6,}" + f" SKILL.md {md_str} tokens" + ) + remaining = len(skills) - _DETAILS_TABLE_LIMIT + if remaining > 0: + lines.append(f" … and {remaining} more") + + return lines + + +def render_context_breakdown_lines( + payload: Dict[str, Any], + *, + details: Optional[Dict[str, Any]] = None, + grid: bool = True, +) -> List[str]: + """Render the full /context view as plain-text lines. + + ``grid=True`` (CLI) prepends the glyph block grid; the gateway passes + ``grid=False`` and keeps its own gauge. ``details`` (from + :func:`compute_context_details`) appends the expanded listings. + """ + lines: List[str] = [] + if grid: + lines.extend(render_context_grid(payload)) + lines.append("") + lines.extend(render_context_category_lines(payload)) + + context_max = int(payload.get("context_max") or 0) + context_used = int(payload.get("context_used") or 0) + if context_max > 0: + pct = int(payload.get("context_percent") or 0) + lines.append("") + lines.append( + f"Context window: {context_used:,} / {context_max:,} tokens ({pct}%)" + ) + + if details is not None: + detail_lines = render_context_details_lines(details) + if detail_lines: + lines.append("") + lines.extend(detail_lines) + else: + lines.append("") + lines.append("Use /context all for per-skill and per-toolset costs.") + return lines diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 65684ced454a..fbb7e6c5e82e 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -25,7 +25,12 @@ import uuid from typing import Any, Dict, List, Optional -from agent.auxiliary_client import call_llm, _is_connection_error, aux_interrupt_protection +from agent.auxiliary_client import ( + AuxiliaryExplicitCancellation, + _is_connection_error, + aux_interrupt_protection, + call_llm, +) from agent.context_engine import ContextEngine, sanitize_memory_context from agent.error_classifier import FailoverReason, classify_api_error from agent.model_metadata import ( @@ -90,9 +95,6 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: HISTORICAL_TASK_HEADING = "## Historical Task Snapshot" -HISTORICAL_IN_PROGRESS_HEADING = "## Historical In-Progress State" -HISTORICAL_PENDING_ASKS_HEADING = "## Historical Pending User Asks" -HISTORICAL_REMAINING_WORK_HEADING = "## Historical Remaining Work" SUMMARY_PREFIX = ( @@ -107,9 +109,7 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: "Topic overlap with the summary does NOT mean you should resume its " "task: even on similar topics, the latest user message WINS. Treat ONLY " "the latest message as the active task and discard stale items from " - f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / " - f"'{HISTORICAL_PENDING_ASKS_HEADING}' / " - f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or " + f"'{HISTORICAL_TASK_HEADING}' entirely — do not 'wrap up' or " "'finish' work described there unless the latest message explicitly " "asks for it. " "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " @@ -142,6 +142,12 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: # "is_compressed_summary" would reach the wire and trip exactly that. COMPRESSED_SUMMARY_METADATA_KEY = "_compressed_summary" COMPRESSED_SUMMARY_HAS_USER_TURN_KEY = "_compressed_summary_has_user_turn" +# Distinguishes rolling micro-compaction markers from batch-compaction +# markers (both carry COMPRESSED_SUMMARY_METADATA_KEY so resume/handoff +# treat them alike). Supersede/defrag/rehydration must only ever touch +# micro markers: a batch marker's content is NOT contained in the micro +# rolling summary, so dropping or rewriting one destroys history. +MICRO_COMPACT_MARKER_KEY = "_micro_compact_marker" _DB_PERSISTED_MARKER = "_db_persisted" _NO_USER_TASK_SENTINEL = "None. This session contains no user-authored turns." @@ -175,6 +181,35 @@ def _fresh_compaction_message_copy(msg: Dict[str, Any]) -> Dict[str, Any]: return fresh +def _template_visible_role(message: Any) -> Optional[str]: + """Role as counted by strict chat-template alternation checks. + + Mistral-family templates (Devstral, Mistral Small 3.x, Magistral) + enforce user/assistant alternation at render time but EXEMPT the tool + flow from the check: ``tool`` results and assistant messages carrying + ``tool_calls`` are skipped. A summary role chosen against the *literal* + neighbouring roles can therefore still violate alternation as the + template sees it. The canonical failure: the protected head ends + ``[user, assistant(tool_calls), tool]``, so the literal last role is + ``tool`` and the summary is pinned to ``role="user"`` -- but the last + role the template counts is ``user``, the template sees user -> user, + and llama.cpp / Mistral-hosted backends reject the ENTIRE request with + a Jinja alternation error (HTTP 500). Because the summary persists in + the stored conversation, every retry replays the same poisoned history + and the session is unrecoverable. + + Returns ``None`` for messages the alternation check skips. + """ + if not isinstance(message, dict): + return None + role = message.get("role") + if role == "tool": + return None + if role == "assistant" and message.get("tool_calls"): + return None + return role + + def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: """Enforce the compaction invariant: no assembled message carries a session-store persistence marker. @@ -219,8 +254,45 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: # stale directive it carried (e.g. "resume exactly from Active Task") survives # embedded in the body and keeps hijacking replies. Keep newest-first; entries # are matched literally. Add a frozen copy here whenever SUMMARY_PREFIX changes. +# NEVER mutate or reorder an existing entry — each one is the exact wire text a +# shipped build persisted, so editing it silently un-normalizes every summary +# written by that build generation; prepend only. tests/agent/ +# test_summary_prefix_semantics.py byte-pins every entry to enforce this. _HISTORICAL_SUMMARY_PREFIXES = ( - # Jul 2026 (#65848 class): identical to the current prefix except it + # Pre-#69619: identical to the current prefix except the stale-item + # discard clause named all four historical headings (the three + # section headers removed by #69619 were still in the template). + # Summaries persisted by builds immediately before #69619 carry this + # exact text and must remain detectable/strippable on resume. + "[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted " + "into the summary below. This is a handoff from a previous context " + "window — treat it as background reference, NOT as active instructions. " + "Do NOT answer questions or fulfill requests mentioned in this summary; " + "they were already addressed. " + "Respond ONLY to the latest user message that appears AFTER this " + "summary — that message is the single source of truth for what to do " + "right now. " + "Topic overlap with the summary does NOT mean you should resume its " + "task: even on similar topics, the latest user message WINS. Treat ONLY " + "the latest message as the active task and discard stale items from " + "'## Historical Task Snapshot' / '## Historical In-Progress State' / " + "'## Historical Pending User Asks' / " + "'## Historical Remaining Work' entirely — do not 'wrap up' or " + "'finish' work described there unless the latest message explicitly " + "asks for it. " + "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " + "back', 'just verify', 'don't do that anymore', 'never mind', a new " + "topic) must immediately end any in-flight work described in the " + "summary; do not re-surface it in later turns. " + "IMPORTANT: Your persistent memory (MEMORY.md, USER.md) in the system " + "prompt is ALWAYS authoritative and active — never ignore or deprioritize " + "memory content due to this compaction note. " + "None of the above restricts HOW you work: your tools remain fully " + "active — keep calling them normally for the active task (edit files, " + "run commands, search) instead of merely narrating what you would do. " + "The current session state (files, config, etc.) may reflect work " + "described here — avoid repeating it:", + # Jul 2026 (#65848 class): identical to the pre-#69619 prefix except it # lacked the explicit "tools remain fully active" clause — the strong # REFERENCE ONLY framing bled into general tool-use suppression # (observed: 7 consecutive narration-only turns immediately after a @@ -236,9 +308,9 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: "Topic overlap with the summary does NOT mean you should resume its " "task: even on similar topics, the latest user message WINS. Treat ONLY " "the latest message as the active task and discard stale items from " - f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / " - f"'{HISTORICAL_PENDING_ASKS_HEADING}' / " - f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or " + "'## Historical Task Snapshot' / '## Historical In-Progress State' / " + "'## Historical Pending User Asks' / " + "'## Historical Remaining Work' entirely — do not 'wrap up' or " "'finish' work described there unless the latest message explicitly " "asks for it. " "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " @@ -303,6 +375,11 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: # itself a context-pressure source and slows every compaction. _SUMMARY_TOKENS_CEILING = 10_000 +# Micro-compaction failure guard: after this many consecutive failures on the +# same cursor position, skip the stuck exchange and advance the cursor so the +# system doesn't busy-loop on an unsummarizable exchange every turn. +_MICRO_COMPACT_MAX_CONSECUTIVE_FAILURES = 3 + # Aggregate cap on the serialized turn block fed to the summarizer prompt # (chars). Per-message truncation (_CONTENT_MAX / _TOOL_ARGS_MAX) alone is # not enough: a compression window with hundreds of already-truncated turns @@ -564,6 +641,12 @@ def _collect_protected_skill_names( # high for small/light tails, but using all 20 as a hard floor here would bring # back the old large-tool-output case where nothing can be compacted. _MAX_TAIL_MESSAGE_FLOOR = 8 + +# Pre-LLM feasibility skip (#60451): when the compressible middle is below +# this fraction of threshold_tokens (and a prior real-usage ineffectiveness +# strike exists), skip the LLM summary call — deterministic dropping alone +# recovers the negligible savings such a summary could deliver. +_FEASIBILITY_SKIP_MIDDLE_FRACTION = 0.10 # Under context pressure (protected-tail tool bodies alone exceed the soft # tail budget), demote large completed tool/file outputs even inside the # protected region — but always keep this many trailing messages verbatim so @@ -692,15 +775,52 @@ def _serialized_length_for_budget(value: Any) -> int: # Responses sessions in particular carry ``codex_reasoning_items`` blobs of # ``encrypted_content`` that can dominate the serialized session (a measured # 214-turn session held ~115K tokens / 27% of its payload there — #55572). +# +# ``reasoning_details`` is handled separately (see +# ``_reasoning_details_text_chars``): its signed/base64 envelope is excluded +# from the budget, mirroring the preflight estimator's exclusion in +# ``model_metadata._estimate_message_tokens_without_images`` (#73298). _REPLAY_BUDGET_KEYS = ( "reasoning", "reasoning_content", - "reasoning_details", "codex_reasoning_items", "codex_message_items", ) +def _reasoning_details_text_chars(value: Any) -> int: + """Textual thinking chars inside a ``reasoning_details`` envelope. + + ``reasoning_details`` carries provider thinking blocks: the actual + thinking TEXT plus opaque signed/base64 envelope blobs (Anthropic + ``signature``, redacted ``data``, encrypted payloads). The envelope is + never billed at anything near chars/4 by the provider and — on every + transport except Codex Responses — is replayed for at most the newest + assistant turn, so charging it on every message inflated the tail-budget + walk and silently shrank the surviving tail (#73298, second site). + + Count only the thinking text (the #51800 lesson: real reasoning text + MUST stay visible to the budget), skip everything else. + """ + if not value: + return 0 + if isinstance(value, str): + return len(value) + total = 0 + if isinstance(value, dict): + value = [value] + if isinstance(value, list): + for part in value: + if isinstance(part, str): + total += len(part) + elif isinstance(part, dict): + for text_key in ("thinking", "text", "summary"): + text = part.get(text_key) + if isinstance(text, str): + total += len(text) + return total + + def _estimate_msg_budget_tokens(msg: dict) -> int: """Token estimate for one message in the tail-protection budget walks. @@ -732,6 +852,17 @@ def _estimate_msg_budget_tokens(msg: dict) -> int: tokens += estimate_tokens_rough(str(tc)) for key in _REPLAY_BUDGET_KEYS: tokens += _serialized_length_for_budget(msg.get(key)) // _CHARS_PER_TOKEN + # reasoning_details: charge only the thinking TEXT, never the signed / + # base64 envelope (#73298 second site; mirrors the preflight estimator's + # exclusion in model_metadata). When the same thinking text already rides + # in ``reasoning``/``reasoning_content`` (measured byte-identical on + # Anthropic-wire sessions), skip it here entirely so the prose is not + # charged twice on top of the envelope exclusion. + if not (msg.get("reasoning") or msg.get("reasoning_content")): + tokens += ( + _reasoning_details_text_chars(msg.get("reasoning_details")) + // _CHARS_PER_TOKEN + ) return tokens @@ -1209,10 +1340,13 @@ def on_session_reset(self) -> None: self._consecutive_timeout_failures = 0 self._last_summary_dropped_count = 0 self._last_summary_fallback_used = False + self._last_feasibility_skip = False self._last_aux_model_failure_error = None self._last_aux_model_failure_model = None self._last_compression_savings_pct = 100.0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 + self._prellm_skip_count = 0 self._fallback_compression_streak = 0 self._verify_compaction_cleared_threshold = False self._last_compression_made_progress = False @@ -1228,6 +1362,15 @@ def on_session_reset(self) -> None: self._active_compression_telemetry = None self._compression_telemetry_seed = None + # Micro-compaction state reset + self._micro_compact_cursor = 0 + self._micro_compact_rolling_summary = "" + self._micro_compact_consecutive_failures = 0 + self._micro_compact_last_failure_cursor = -1 + self._micro_compact_passes = 0 + self._micro_compact_tokens_saved_total = 0 + self._micro_compact_turns_since_pass = 0 + def _begin_compression_telemetry( self, *, @@ -1255,6 +1398,7 @@ def _begin_compression_telemetry( "protected_head_tokens": None, "protected_tail_tokens": None, "middle_window_tokens": None, + "prellm_skip_count": 0, "aux_prompt_tokens": None, "aux_output_reservation": None, "aux_provider": "", @@ -1321,6 +1465,125 @@ def _record_aux_compression_call( previous = telemetry.get("aux_call_duration_ms") or 0 telemetry["aux_call_duration_ms"] = previous + max(0, int(duration_ms)) + def _emit_init_summary_once(self) -> None: + """Emit the informative startup line once, on first resolution. + + Deferred out of ``__init__`` (#32221): the line reports resolved token + budgets, so emitting it there would force the synchronous + ``get_model_context_length()`` probe during construction. Reads via + the properties below are safe here because + ``_resolved_context_length`` is already set. + """ + if not getattr(self, "_log_init_summary", False): + return + self._log_init_summary = False + logger.info( + "Context compressor initialized: model=%s context_length=%d " + "threshold=%d (%.0f%%) target_ratio=%.0f%% tail_budget=%d " + "provider=%s base_url=%s", + self.model, self._resolved_context_length, self.threshold_tokens, + self.threshold_percent * 100, self.summary_target_ratio * 100, + self.tail_token_budget, + self.provider or "none", self.base_url or "none", + ) + + def _resolve_context_length(self) -> int: + """Resolve and cache the model's context length on first access.""" + if self._resolved_context_length is None: + self._resolved_context_length = get_model_context_length( + self.model, + base_url=self.base_url, + api_key=self.api_key, + config_context_length=self._config_context_length, + provider=self.provider, + ) + # Small-context threshold floor: models under 512K trigger at + # >=75% so compaction doesn't fire with half the window still + # free. Raise-only; must run AFTER context_length is resolved + # and BEFORE threshold_tokens is derived (deferred here from + # __init__ along with the resolution itself, #32221). + # _base_threshold_percent already has the per-model override + # applied, so the floor stacks on top of it. + self.threshold_percent = self._effective_threshold_percent( + self._resolved_context_length, self._base_threshold_percent, + ) + self._emit_init_summary_once() + return self._resolved_context_length + + @property + def context_length(self) -> int: + return self._resolve_context_length() + + @context_length.setter + def context_length(self, value: int) -> None: + # No-op guard: repeated assignment of the SAME window (e.g. the codex + # app-server usage callback re-reports the window on every response) + # must not invalidate the derived budgets — that would wipe runtime + # corrections applied directly to threshold_tokens/tail_token_budget + # (see conversation_compression's aux-context threshold sync), which + # persisted on main's eager-init behavior. + if value == getattr(self, "_resolved_context_length", None): + return + self._resolved_context_length = value + # Re-apply the small-context floor (raise-only) for the genuinely new + # window so the invalidated budgets below recompute coherently — + # percent and tokens must derive from the same window. Skipped on + # bare test instances built via object.__new__ that never ran + # __init__ (no _base_threshold_percent). + _base = getattr(self, "_base_threshold_percent", None) + if _base is not None: + self.threshold_percent = self._effective_threshold_percent( + value, _base, + ) + self._threshold_tokens = None + self._tail_token_budget = None + self._max_summary_tokens = None + self._emit_init_summary_once() + + @property + def threshold_tokens(self) -> int: + if self._threshold_tokens is None: + # Resolve the window FIRST (may apply the small-context floor to + # threshold_percent as a side effect) so the percent read below + # is the floored value regardless of argument evaluation order. + _ctx = self.context_length + # Floor: never compress below MINIMUM_CONTEXT_LENGTH tokens even + # if the percentage would suggest a lower value (#14690 handles + # the degenerate small-window case inside the helper). + self._threshold_tokens = self._compute_threshold_tokens( + _ctx, self.threshold_percent, self.max_tokens, + ) + # Apply absolute token cap (compression.threshold_tokens) — + # takes the lower of the ratio-based threshold and the cap. + self._apply_threshold_tokens_cap() + return self._threshold_tokens + + @threshold_tokens.setter + def threshold_tokens(self, value: int) -> None: + self._threshold_tokens = value + + @property + def tail_token_budget(self) -> int: + if self._tail_token_budget is None: + self._tail_token_budget = int(self.threshold_tokens * self.summary_target_ratio) + return self._tail_token_budget + + @tail_token_budget.setter + def tail_token_budget(self, value: int) -> None: + self._tail_token_budget = value + + @property + def max_summary_tokens(self) -> int: + if self._max_summary_tokens is None: + self._max_summary_tokens = min( + int(self.context_length * 0.05), _SUMMARY_TOKENS_CEILING, + ) + return self._max_summary_tokens + + @max_summary_tokens.setter + def max_summary_tokens(self, value: int) -> None: + self._max_summary_tokens = value + def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> None: """Clear all per-session compaction state at a real session boundary. @@ -1346,10 +1609,13 @@ def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> Non self._consecutive_timeout_failures = 0 self._last_summary_dropped_count = 0 self._last_summary_fallback_used = False + self._last_feasibility_skip = False self._last_aux_model_failure_error = None self._last_aux_model_failure_model = None self._last_compression_savings_pct = 100.0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 + self._prellm_skip_count = 0 self._fallback_compression_streak = 0 self._verify_compaction_cleared_threshold = False self._last_compression_made_progress = False @@ -1376,6 +1642,8 @@ def bind_session_state(self, session_db: Any = None, session_id: str = "") -> No self._consecutive_timeout_failures = 0 self._fallback_compression_streak = 0 self._ineffective_compression_count = 0 + self._prellm_skip_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self.get_active_compression_failure_cooldown() self._load_fallback_compression_streak() self._load_ineffective_compression_count() @@ -1519,9 +1787,34 @@ def _record_ineffective_compression_verdict(self, count: int) -> None: self._ineffective_compression_count = count self._persist_ineffective_compression_count() - def record_completed_compaction(self, *, used_fallback: bool = False) -> None: - """Record one completed boundary and its summary quality.""" + def record_completed_compaction( + self, *, used_fallback: bool = False, feasibility_skip: bool = False, + ) -> None: + """Record one completed boundary and its summary quality. + + ``feasibility_skip=True`` marks a deliberate pre-LLM skip (#60451): + the boundary is streak-NEUTRAL for ``_fallback_compression_streak`` + (neither incremented nor reset). It still arms the real-usage + effectiveness verdict (``_verify_compaction_cleared_threshold``) on + purpose — a skipped-summary drop that fails to clear the threshold is + exactly the incompressible-transcript case the ineffective-strike + breaker exists for, and its recovery probe bounds the block. + """ self._verify_compaction_cleared_threshold = True + if feasibility_skip: + # A deliberate pre-LLM feasibility skip (#60451) is not a + # summary-quality verdict: it must neither extend a fallback + # streak (two skips would otherwise latch the >= 2 breaker and + # disable compression entirely — including the cheap deterministic + # dropping the skip exists to reach) nor reset one (a skip proves + # nothing about the summary model's health). + if not self.quiet_mode: + logger.info( + "Compaction completed via pre-LLM feasibility skip; " + "fallback_compression_streak unchanged (%d)", + self._fallback_compression_streak, + ) + return if used_fallback: self._fallback_compression_streak += 1 if not self.quiet_mode: @@ -1540,6 +1833,11 @@ def get_active_compression_failure_cooldown( refresh: bool = False, ) -> Optional[Dict[str, Any]]: """Return the live compression-failure cooldown for the bound session.""" + if refresh: + # Transaction rollback must distinguish an authoritative empty row + # from a failed/unavailable durable read. The public return value + # cannot do so because it deliberately falls back to local state. + self._last_cooldown_refresh_was_authoritative = None now_mono = time.monotonic() local_state = None if self._summary_failure_cooldown_until > now_mono: @@ -1564,10 +1862,16 @@ def get_active_compression_failure_cooldown( try: state = getter(session_id) except sqlite3.Error as exc: + if refresh: + self._last_cooldown_refresh_was_authoritative = False logger.debug("compression failure cooldown lookup failed: %s", exc) return local_state except Exception: + if refresh: + self._last_cooldown_refresh_was_authoritative = False return local_state + if refresh: + self._last_cooldown_refresh_was_authoritative = True if not state: if refresh: if local_state is not None and self._cooldown_persist_failed: @@ -1628,7 +1932,43 @@ def _record_compression_failure_cooldown( self._cooldown_persist_failed = True logger.debug("compression failure cooldown persist failed (non-sqlite): %s", exc) + def record_timeout_failure(self, error: str) -> None: + """Record a consecutive timeout failure using the shared cooldown ladder. + + Used by both the summary-LLM exception handler (inline at line ~3714) + and the host-level ``compress_context`` timeout wrapper in + ``run_compress_context_with_progress_timeout``. Avoids re-implementing + the ladder at each call site (#62452). + """ + _TIMEOUT_COOLDOWN_LADDER = (60, 300, 900) + self._consecutive_timeout_failures = ( + getattr(self, "_consecutive_timeout_failures", 0) + 1 + ) + cooldown = _TIMEOUT_COOLDOWN_LADDER[ + min(self._consecutive_timeout_failures, + len(_TIMEOUT_COOLDOWN_LADDER)) - 1 + ] + self._record_compression_failure_cooldown(float(cooldown), error) + def _clear_compression_failure_cooldown(self) -> None: + # #76354 review F4: fence check BEFORE cooldown-clear. A late worker + # whose host already timed out (and recorded a timeout cooldown) must + # not undo that cooldown when its summary eventually succeeds. The + # hook is installed by compress_context for the duration of the + # fenced call; when it reports cancellation, keep the host's cooldown. + cancelled_check = getattr(self, "_compression_cancelled_check", None) + if callable(cancelled_check): + try: + if cancelled_check(): + logger.info( + "Skipping compression cooldown clear: host already " + "cancelled this compression attempt" + ) + return + except Exception: + logger.debug( + "compression cancellation check failed", exc_info=True + ) self._summary_failure_cooldown_until = 0.0 self._last_summary_error = None self._consecutive_timeout_failures = 0 @@ -1731,6 +2071,7 @@ def update_model( # trigger invalidates them. Keep the durable copy in sync so a # restart doesn't resurrect strikes this recalibration just voided. self._record_ineffective_compression_verdict(0) + self._prellm_skip_count = 0 if runtime_changed: self._fallback_compression_streak = 0 self._persist_fallback_compression_streak() @@ -1747,6 +2088,15 @@ def update_model( # rationale as the gpt-5.5/Codex 85% autoraise. _MIN_CTX_TRIGGER_RATIO = 0.85 + # Anti-thrash recovery window (#14694): once the ineffective/fallback + # breaker trips, automatic compaction stays blocked for this long, then + # ONE probe attempt is allowed (counters drop to 1 strike, so another + # ineffective pass re-trips immediately). Long enough that a genuinely + # incompressible session isn't compacting in a loop; short enough that a + # session which has since grown real compressible material recovers well + # before it rides into the provider's hard context limit. + _ANTI_THRASH_RECOVERY_SECONDS = 300.0 + @staticmethod def _coerce_max_tokens(value: Any) -> int | None: """Normalize a max_tokens value to a positive int or None. @@ -1944,56 +2294,53 @@ def __init__( # deterministic "summary unavailable" handoff and drop the middle window. self.abort_on_summary_failure = abort_on_summary_failure - self.context_length = get_model_context_length( - model, base_url=base_url, api_key=api_key, - config_context_length=config_context_length, - provider=provider, - ) - # Small-context threshold floor: models under 512K trigger at >=75% - # so compaction doesn't fire with half the window still free (the - # incompressible floor makes 50%-triggered compaction thrash on - # 128K-262K models). Raise-only; must run AFTER context_length is - # resolved and BEFORE threshold_tokens is derived. The pre-floor - # value is kept so update_model() can re-derive for a new window - # (switching small -> large must drop back to the configured value). - # Note: _base_threshold_percent already has the per-model override - # applied, so the floor stacks on top of any model-specific threshold. + # ── Micro-compaction (per-turn rolling compaction) ───────── + # Default: OFF. Each pass rewrites already-sent history, so it breaks + # the prompt-cache prefix every turn instead of at an episodic + # boundary. Operators opt in via `compression.micro_compact: true`. + self._micro_compact_enabled: bool = False + self._micro_compact_cursor: int = 0 + self._micro_compact_rolling_summary: str = "" + self._micro_compact_consecutive_failures: int = 0 + self._micro_compact_last_failure_cursor: int = -1 + self._micro_compact_defrag_threshold_tokens: int = 2000 + # Set by _defrag_rolling_summary when it pops _DB_PERSISTED_MARKER + # from a live dict in place; consumed by finalize_turn to invalidate + # the agent's bounded flush-scan cursor (sibling of the #75170 site). + self._flush_scan_cursor_invalidated: bool = False + self._micro_compact_passes: int = 0 + self._micro_compact_tokens_saved_total: int = 0 + # Cadence: run a pass every Nth completed turn. Each pass rewrites + # already-sent history and so breaks the prompt-cache prefix, which + # makes this the dial that sets how often that break is paid. 1 = + # every turn (most aggressive reclaim, one break per turn). + self._micro_compact_every_n_turns: int = 1 + self._micro_compact_turns_since_pass: int = 0 + + # Defer context-length resolution to first access (#32221): + # get_model_context_length() can issue a synchronous /models HTTP + # probe, which must not block AIAgent construction. The small-context + # threshold floor and the absolute threshold cap both need the + # resolved window, so they are applied on first resolution (see + # _resolve_context_length / the threshold_tokens property) instead + # of here. update_model() re-derives the floor for a new window from + # _config_threshold_percent (the raw config value snapshotted above), + # so switching small -> large correctly drops back to the configured + # value. + self._config_context_length = config_context_length self._configured_threshold_percent = self.threshold_percent - self.threshold_percent = self._effective_threshold_percent( - self.context_length, self._base_threshold_percent, - ) - threshold_percent = self.threshold_percent - # Floor: never compress below MINIMUM_CONTEXT_LENGTH tokens even if - # the percentage would suggest a lower value. This prevents premature - # compression on large-context models at 50% while keeping the % sane - # for models right at the minimum. _compute_threshold_tokens also - # guards the degenerate case where the floor would equal/exceed the - # window (small models), so auto-compression can still fire (#14690). - self.threshold_tokens = self._compute_threshold_tokens( - self.context_length, threshold_percent, self.max_tokens, - ) - # Apply absolute token cap (compression.threshold_tokens) — takes - # the lower of the ratio-based threshold and the cap. - self._apply_threshold_tokens_cap() + self._resolved_context_length: int | None = None + self._threshold_tokens: int | None = None + self._tail_token_budget: int | None = None + self._max_summary_tokens: int | None = None self.compression_count = 0 - # Derive token budgets: ratio is relative to the threshold, not total context - target_tokens = int(self.threshold_tokens * self.summary_target_ratio) - self.tail_token_budget = target_tokens - self.max_summary_tokens = min( - int(self.context_length * 0.05), _SUMMARY_TOKENS_CEILING, - ) - - if not quiet_mode: - logger.info( - "Context compressor initialized: model=%s context_length=%d " - "threshold=%d (%.0f%%) target_ratio=%.0f%% tail_budget=%d " - "provider=%s base_url=%s", - model, self.context_length, self.threshold_tokens, - threshold_percent * 100, self.summary_target_ratio * 100, - self.tail_token_budget, - provider or "none", base_url or "none", - ) + # The "initialized" log reports resolved token budgets, which would + # force the deferred get_model_context_length() probe to run inside + # __init__ and re-introduce the exact synchronous blocking this change + # removes (#32221). Emit it on first context-length resolution instead + # so construction stays non-blocking on every path (not just quiet). + self._log_init_summary = not quiet_mode self._context_probed = False # True after a step-down from context error self.last_prompt_tokens = 0 @@ -2016,6 +2363,15 @@ def __init__( # Anti-thrashing: track whether last compression was effective self._last_compression_savings_pct: float = 100.0 self._ineffective_compression_count: int = 0 + # Monotonic deadline after which a tripped anti-thrash guard grants + # one probation probe (#14694). 0.0 = clock not armed. Armed lazily on + # the first blocked evaluation; deliberately NOT durable, so a process + # restart with a persisted tripped counter (#69872) waits a full fresh + # window before probing (#54923: restart must never disarm a guard). + self._anti_thrash_recovery_deadline: float = 0.0 + # Pre-LLM feasibility skips (#60451). Observability only; NEVER feeds + # the ineffectiveness strike latch or the fallback streak breaker. + self._prellm_skip_count: int = 0 # Consecutive completed deterministic-fallback boundaries. Unlike the # real-usage effectiveness counter, ordinary fitting responses must not # reset this breaker; only a healthy completed summary does. @@ -2037,6 +2393,7 @@ def __init__( # (gateway hygiene, /compress) can surface a visible warning. self._last_summary_dropped_count: int = 0 self._last_summary_fallback_used: bool = False + self._last_feasibility_skip: bool = False # When summary generation fails we now ABORT compression entirely # and return the original messages unchanged instead of dropping # the middle window with a static placeholder. Callers inspect @@ -2306,21 +2663,66 @@ def _automatic_compression_blocked_locally(self) -> bool: _cooldown_remaining, ) return True - # Anti-thrashing: back off if recent compressions were ineffective + # Anti-thrashing: back off if recent compressions were ineffective. + # The back-off must not be permanent (#14694): the tripped state was + # judged against the transcript as it existed THEN (e.g. a middle + # region too small to matter), but the conversation keeps growing and + # can accumulate plenty of compressible material later. Without a + # recovery path the session never auto-compacts again and rides into + # the provider's hard context limit. Recovery is a probation probe: + # after _ANTI_THRASH_RECOVERY_SECONDS of continuous block, allow ONE + # attempt by dropping the tripped counter(s) to 1 strike (persisted, + # so sibling agents on the same session row unblock too). If the probe + # is ineffective again the very next verdict re-trips the guard, so + # the worst case in the truly-incompressible state is one compaction + # attempt per recovery window — bounded, not thrash. + # + # The clock is armed lazily on the first BLOCKED evaluation rather + # than persisted at trip time: a fresh process that loads a durable + # tripped counter (#69872) therefore starts a full window blocked, + # preserving the restart-must-not-disarm contract (#54923). if ( self._ineffective_compression_count >= 2 or self._fallback_compression_streak >= 2 ): + _now = time.monotonic() + if self._anti_thrash_recovery_deadline <= 0.0: + self._anti_thrash_recovery_deadline = ( + _now + self._ANTI_THRASH_RECOVERY_SECONDS + ) + elif _now >= self._anti_thrash_recovery_deadline: + self._anti_thrash_recovery_deadline = 0.0 + if self._ineffective_compression_count >= 2: + self._record_ineffective_compression_verdict(1) + if self._fallback_compression_streak >= 2: + self._fallback_compression_streak = 1 + self._persist_fallback_compression_streak() + if not self.quiet_mode: + logger.info( + "Anti-thrashing recovery: %.0fs elapsed since the " + "guard tripped — allowing one compaction probe " + "(ineffective=%d fallback=%d).", + self._ANTI_THRASH_RECOVERY_SECONDS, + self._ineffective_compression_count, + self._fallback_compression_streak, + ) + return False if not self.quiet_mode: logger.warning( "Compression skipped — repeated compaction attempts did not " "restore healthy context. ineffective=%d fallback=%d. " - "Consider /new to start fresh, or /compress for " - "focused compression.", + "Auto-compaction will retry once in %.0fs. Consider /new " + "to start fresh, or /compress for focused " + "compression.", self._ineffective_compression_count, self._fallback_compression_streak, + max(0.0, self._anti_thrash_recovery_deadline - _now), ) return True + # Guard not tripped (counters were cleared by an effective compaction + # or a fitting real-usage reading) — disarm any pending recovery clock + # so a LATER trip starts its own full window. + self._anti_thrash_recovery_deadline = 0.0 return False # ------------------------------------------------------------------ @@ -2968,12 +3370,6 @@ def _bullets(items: list[str], limit: int = 8) -> str: ## Active State Unknown from deterministic fallback. Inspect current repository/session state if needed. -{HISTORICAL_IN_PROGRESS_HEADING} -Unknown from deterministic fallback — the latest user ask is recorded once under -"{HISTORICAL_TASK_HEADING}" above as historical context only. Do NOT treat it as an -unfulfilled instruction to re-answer; verify current state and continue from the -protected recent messages after this summary. - ## Blocked {_bullets(blockers, limit=5)} @@ -2983,17 +3379,9 @@ def _bullets(items: list[str], limit: int = 8) -> str: ## Resolved Questions None recoverable from deterministic fallback. -{HISTORICAL_PENDING_ASKS_HEADING} -None recoverable from deterministic fallback. (The latest user ask is preserved once -under "{HISTORICAL_TASK_HEADING}" as historical context — it is NOT necessarily -outstanding.) - ## Relevant Files {_bullets(relevant_files, limit=12)} -{HISTORICAL_REMAINING_WORK_HEADING} -Continue from the most recent unfulfilled user ask and protected tail messages. Verify state with tools before making claims. - ## Last Dropped Turns {_bullets(last_dropped_turns, limit=8)} @@ -3304,9 +3692,6 @@ def _generate_summary( - Any running processes or servers - Environment details that matter] -{HISTORICAL_IN_PROGRESS_HEADING} -[Work currently underway — what was being done when compaction fired] - ## Blocked [Any blockers, errors, or issues not yet resolved. Include exact error messages.] @@ -3316,15 +3701,9 @@ def _generate_summary( ## Resolved Questions {_resolved_questions_instructions} -{HISTORICAL_PENDING_ASKS_HEADING} -{_pending_asks_instructions} - ## Relevant Files [Files read, modified, or created — with brief note on each] -{HISTORICAL_REMAINING_WORK_HEADING} -[What remains to be done — framed as STALE context for reference only. The agent must NOT resume this work unless the latest user message explicitly asks for it.] - ## Critical Context [Any specific values, error messages, configuration details, or data that would be lost without explicit preservation. NEVER include API keys, tokens, passwords, or credentials — write [REDACTED] instead.] @@ -4003,6 +4382,12 @@ def _find_context_summaries( end: int, ) -> list[tuple[int, str]]: """Find handoff summaries inside a compression window.""" + n = len(messages) + # Defensive: clamp bounds so a caller passing an out-of-range end + # (e.g. tail-cut returning len(messages)+1 when head_end >= n) + # cannot trigger IndexError. (#75588) + start = max(0, min(start, n)) + end = max(start, min(end, n)) summaries: list[tuple[int, str]] = [] for idx in range(start, end): content = messages[idx].get("content") @@ -4772,7 +5157,7 @@ def _find_tail_cut_by_tokens( # exists to prevent. Re-align FORWARD (never backward, which would give # the floor's message back) so a raised cut skips to the end of the # group and the whole call/result pair is summarised together. - return self._align_boundary_forward(messages, max(cut_idx, head_end + 1)) + return min(n, self._align_boundary_forward(messages, max(cut_idx, head_end + 1))) # ------------------------------------------------------------------ # ContextEngine: manual /compress preflight @@ -4793,6 +5178,759 @@ def has_content_to_compress(self, messages: List[Dict[str, Any]]) -> bool: # Main compression entry point # ------------------------------------------------------------------ + def _resolve_compact_cursor( + self, + messages: List[Dict[str, Any]], + head_end: int, + tail_start: int, + ) -> int: + """Derive the micro-compaction cursor from in-memory state or transcript scan. + + Returns the index of the first message that has NOT yet been absorbed + into the rolling summary. If the in-memory cursor ``_micro_compact_cursor`` + is valid (non-zero and within the compressible window), use it directly. + Otherwise scan from *head_end* through *tail_start* for the last context + summary marker and set the cursor past it. + """ + if self._micro_compact_cursor > head_end and self._micro_compact_cursor < tail_start: + return self._micro_compact_cursor + # Scan transcript for the last summary marker + last_summary_idx = -1 + for idx in range(head_end, tail_start): + if self._is_context_summary_message(messages[idx]): + last_summary_idx = idx + if last_summary_idx >= head_end: + cursor = last_summary_idx + 1 + # Resumed session: in-memory state is gone but the marker survives. + # Carry its text forward so the next pass merges into the existing + # history instead of replacing it with a single-exchange summary. + if not self._micro_compact_rolling_summary.strip(): + recovered = self._rolling_summary_from_marker( + messages[last_summary_idx].get("content") + ) + if recovered: + self._micro_compact_rolling_summary = recovered + # Rehydration is containment proof: this marker's text now + # lives inside the rolling summary, so it becomes + # supersede/defrag-eligible. This also covers a BATCH + # marker adopted as the rolling base after a batch + # compaction reset — safe precisely because we just + # absorbed its content. Markers whose content we did NOT + # absorb never get the key and are never dropped. + messages[last_summary_idx][MICRO_COMPACT_MARKER_KEY] = True + logger.info( + "Micro-compaction: recovered rolling summary from " + "transcript (%d chars)", len(recovered), + ) + else: + cursor = head_end + self._micro_compact_cursor = cursor + return cursor + + def _find_one_exchange( + self, + messages: List[Dict[str, Any]], + start: int, + tail_start: int, + ) -> Optional[tuple[int, int]]: + """Find the next complete exchange starting at *start*. + + An exchange is one full agent turn: the first assistant message after + *start* plus everything through the end of that turn — tool results + and any follow-up assistant iterations — up to (exclusive) the next + ``user`` message. Returns ``(exchange_start, exchange_end)`` indices + into *messages*, or ``None`` if no complete, safely-spliceable turn is + available before *tail_start*. + + The full-turn shape is an alternation-safety requirement, not a + convenience: the splice replaces the span with a single + ``assistant``-role summary marker, so the span must be bounded by + user messages on the right (``messages[exchange_end]`` is ``user``). + Absorbing only the first assistant+tools group of a multi-iteration + turn would leave the marker adjacent to the turn's next assistant + message — two consecutive assistant turns, which strict providers + reject and ``repair_message_sequence`` would then mangle. + + User messages are deliberately NOT part of an exchange. The walk skips + past them to reach the assistant message, and ``exchange_start`` is that + assistant index, so user turns are never absorbed into the rolling + summary and their text stays verbatim for the life of the session. + This is the intended behaviour, not an oversight: what the assistant + emits is largely an account of what it did, which survives summarising, + while the user's own words are the instructions everything else is + derived from and are the one thing that cannot be reconstructed from + context. They are also cheap — a prompt is normally a tiny fraction + of the tokens a single tool result costs. + """ + idx = start + n = len(messages) + if idx >= n or idx >= tail_start: + return None + + # Walk past user messages and existing summary markers until we hit a + # real assistant message with actual output (content or tool_calls). + # Summary markers are assistant-role themselves, so they must be + # skipped explicitly or a rehydrated cursor could try to absorb the + # marker that carries the compacted history. + while idx < tail_start and idx < n: + msg = messages[idx] + if msg.get("role") == "assistant" and not self._is_context_summary_message(msg): + break + idx += 1 + + if idx >= tail_start or idx >= n: + return None + + exchange_start = idx + + # Consume the full turn: assistant / tool messages until the next + # user message (or an existing summary marker) ends the turn. + idx += 1 + while idx < tail_start and idx < n: + msg = messages[idx] + role = msg.get("role") + if role not in ("assistant", "tool"): + break + if self._is_context_summary_message(msg): + break + idx += 1 + + if idx <= exchange_start: + return None + + # Splice-boundary guard: the message right after the exchange must + # close the turn. If the walk stopped because it ran into + # *tail_start* mid-turn (boundary is assistant or tool — including + # an assistant-role summary marker), splicing here would leave the + # assistant-role marker adjacent to the turn's remaining + # assistant/tool messages — invalid alternation. Skip this pass; the + # tail recedes as the conversation grows and the turn becomes + # absorbable later. Any other boundary role (user, or a stray + # system/injected message) is a safe splice point — the marker is + # assistant-role, so no same-role adjacency is possible — and + # accepting them keeps one odd message from wedging the cursor + # forever. + if idx >= n: + return None + boundary = messages[idx] + if not isinstance(boundary, dict) or boundary.get("role") in ("assistant", "tool"): + return None + return (exchange_start, idx) + + def _serialize_one_exchange( + self, + messages: List[Dict[str, Any]], + start: int, + end: int, + ) -> str: + """Serialize a single exchange for the micro-summarizer. + + Delegates to the batch path's ``_serialize_for_summary`` (same + truncation, redaction, think-block stripping, and media labeling), + scoped to one exchange — one serializer, one place to fix. + """ + return self._serialize_for_summary(messages[start:end]) + + def _build_micro_summary_prompt( + self, + existing_summary: str, + exchange_text: str, + ) -> List[Dict[str, str]]: + """Build the prompt messages for a single-exchange micro-summary.""" + if existing_summary.strip(): + summary_block = existing_summary + else: + summary_block = "(No previous summary yet.)" + + user_prompt = ( + "You are a summarization agent creating a compact record of an " + "ongoing conversation. You are given a running summary and the " + "next exchange from the conversation. Merge the exchange's key " + "decisions, requirements, file paths, and open questions into the " + "summary. Preserve the summary's structure. Drop resolved details " + "that are no longer relevant. Add new decisions, file paths, and " + "open questions.\n\n" + "NEVER include API keys, tokens, passwords, secrets, credentials, " + "or connection strings in the summary \u2014 replace any that appear " + f"with [REDACTED].\n\n" + f"## Current Running Summary\n{summary_block}\n\n" + f"## Next Exchange to Merge\n{exchange_text}\n\n" + "Return ONLY the updated summary text, no preamble or explanation. " + "Do not include this instruction block in your output." + ) + + return [ + {"role": "system", "content": "You are a conversation summarization assistant."}, + {"role": "user", "content": user_prompt}, + ] + + def _micro_summarize_one( + self, + exchange_text: str, + ) -> Optional[str]: + """Micro-summarize one exchange into the rolling summary via aux LLM. + + Calls the same auxiliary compression model as the batch path, with + a focused prompt that merges one exchange into the running summary. + Returns the updated summary text, or ``None`` on failure. + """ + from agent.auxiliary_client import call_llm, aux_interrupt_protection + + messages = self._build_micro_summary_prompt( + self._micro_compact_rolling_summary, + exchange_text, + ) + + call_kwargs = { + "task": "compression", + "messages": messages, + "max_tokens": min(1500, self.max_summary_tokens or 1500), + "temperature": 0.1, + } + if self.summary_model: + call_kwargs["model"] = self.summary_model + if self.model: + call_kwargs.setdefault("main_runtime", { + "model": self.model, + "provider": self.provider or "", + "base_url": self.base_url or "", + "api_key": self.api_key or "", + "api_mode": getattr(self, "api_mode", "") or "", + }) + + try: + with aux_interrupt_protection(): + response = call_llm(**call_kwargs) + except Exception as exc: + logger.info("micro-summarization call failed: %s", exc) + return None + + message = response.choices[0].message + if isinstance(message, dict): + content = message.get("content") + else: + content = getattr(message, "content", message) + if not isinstance(content, str): + content = str(content) if content else "" + content = content.strip() + if not content: + logger.info("micro-summarization returned empty content") + return None + + from agent.agent_runtime_helpers import strip_think_blocks + stripped = strip_think_blocks(None, content).strip() + return stripped if stripped else None + + def _needs_defrag(self) -> bool: + """Return True when the rolling summary is large enough to defrag.""" + content_tokens = estimate_tokens_rough(self._micro_compact_rolling_summary) + return content_tokens >= self._micro_compact_defrag_threshold_tokens + + def _defrag_rolling_summary( + self, + messages: List[Dict[str, Any]], + ) -> bool: + """Re-summarize the rolling summary TEXT and rewrite the marker in place. + + Merging exchange after exchange makes the rolling summary baggy — + repetitive, and larger than the material justifies. Defrag compacts + the summary *itself*: one aux call over the accumulated summary text, + then the existing marker's content is rewritten in place. + + Deliberately transcript-shape-neutral: no messages are spliced, no + user turns are touched, and the cursor does not move. The original + implementation serialized the whole remaining middle (user turns + included) and spliced it into the marker, which silently absorbed + user messages — violating the feature's core "your messages are never + compacted" invariant. Un-absorbed exchanges stay where they are and + get absorbed by later per-exchange passes. + + Returns True when a pass actually rewrote the summary. + """ + old_summary = self._micro_compact_rolling_summary + if not old_summary.strip(): + return False + # Feed the old summary through the merge prompt with an empty base: + # "merge these decisions into (no previous summary)" is exactly a + # rewrite-compactly instruction for the accumulated text. + self._micro_compact_rolling_summary = "" + fresh_summary = self._micro_summarize_one(old_summary) + if not fresh_summary: + self._micro_compact_rolling_summary = old_summary + return False + self._micro_compact_rolling_summary = fresh_summary + # Rewrite the newest MICRO marker's content in place so the transcript + # and the in-memory summary stay in step (resume rehydrates from it). + # Scoped to micro-tagged markers: rewriting a batch-compaction marker + # would overwrite history the rolling summary does not contain. + for idx in range(len(messages) - 1, -1, -1): + entry = messages[idx] + if ( + isinstance(entry, dict) + and entry.get(COMPRESSED_SUMMARY_METADATA_KEY) + and entry.get(MICRO_COMPACT_MARKER_KEY) + ): + entry["content"] = self._render_micro_marker_content(fresh_summary) + # Content changed after a possible flush — clear the persisted + # stamp so the DB sync/flush rewrites the row. + entry.pop(_DB_PERSISTED_MARKER, None) + # Sibling of the finalize_turn pop site (#75170): this pop + # also strips the marker from a LIVE dict in place, so the + # bounded flush-scan cursor would identity-skip the rewritten + # marker and the defragged summary would never reach state.db. + # The compressor holds no agent reference, so raise a flag the + # finalizer consumes to invalidate agent._db_flush_scan_prefix. + # (The pop sites at module scope — fresh copies in + # strip-marker helpers — break identity and need no flag.) + self._flush_scan_cursor_invalidated = True + break + logger.info( + "Micro-compaction defrag: rolling summary re-summarized " + "(%d -> %d chars)", len(old_summary), len(fresh_summary), + ) + return True + + def _micro_compact( + self, + messages: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """Run one round of micro-compaction on the conversation. + + Absorbs the oldest uncompacted exchange into the rolling summary, + advancing the in-memory cursor. Runs in post-turn idle time. + + This is the public entry point called from ``finalize_turn()``. + Returns the (possibly modified) message list. + + NOTE: the in-memory splice alone is not persisted — the subsequent + ``_persist_session`` flush is append-only, so old DB rows stay + ``active=1`` and a session resume double-loads both the summary and + the original exchanges. This method therefore also calls + ``archive_and_compact`` on the session DB to soft-archive old rows + and insert the compacted set atomically. + """ + if not self._micro_compact_enabled: + return messages + + # Cadence gate. A pass rewrites already-sent history, so it costs one + # prompt-cache break; `every_n_turns` is how an operator trades reclaim + # frequency against that cost. Counted per invocation rather than per + # committed pass so a turn that finds nothing to absorb still advances + # the cadence and cannot wedge it. + every_n = max(1, int(self._micro_compact_every_n_turns or 1)) + if every_n > 1: + self._micro_compact_turns_since_pass += 1 + if self._micro_compact_turns_since_pass < every_n: + return messages + self._micro_compact_turns_since_pass = 0 + + n_messages = len(messages) + if n_messages < 4: + return messages + + head_size = self._protect_head_size(messages) + compress_start = self._align_boundary_forward(messages, head_size) + compress_end = self._find_tail_cut_by_tokens(messages, compress_start) + + if compress_start >= compress_end: + return messages + + cursor = self._resolve_compact_cursor(messages, compress_start, compress_end) + if cursor >= compress_end: + return messages + + # Find the next exchange + exchange = self._find_one_exchange(messages, cursor, compress_end) + if exchange is None: + return messages + + exchange_start, exchange_end = exchange + + # Baseline for telemetry. Taken only once an exchange is in hand, so + # turns that no-op early don't pay for the scan. + _started_at = time.monotonic() + _tokens_before = estimate_messages_tokens_rough(messages) + _messages_before = n_messages + + def _elapsed_ms() -> int: + return int((time.monotonic() - _started_at) * 1000) + + # Check for defrag trigger: the rolling summary itself has grown + # baggy. Defrag rewrites the summary text and the existing marker in + # place — no splice, no cursor movement, no user turns touched — so + # the transcript shape is unchanged and this pass does not also + # absorb an exchange (one aux call per turn either way). + if self._needs_defrag(): + defragged = self._defrag_rolling_summary(messages) + if defragged: + self._sync_micro_compact_to_db(messages) + self._micro_compact_consecutive_failures = 0 + self._micro_compact_last_failure_cursor = -1 + self._emit_micro_compaction_telemetry( + outcome="defrag" if defragged else "defrag_failed", + messages_before=_messages_before, + messages_after=len(messages), + tokens_before=_tokens_before, + tokens_after=estimate_messages_tokens_rough(messages), + duration_ms=_elapsed_ms(), + ) + return messages + + # Whether this pass's summary will be cumulative — i.e. whether it + # subsumes any earlier marker. Captured before summarizing. + _cumulative = bool(self._micro_compact_rolling_summary.strip()) + + # Micro-summarize one exchange + exchange_text = self._serialize_one_exchange(messages, exchange_start, exchange_end) + _exchange_tokens = estimate_tokens_rough(exchange_text) + updated_summary = self._micro_summarize_one(exchange_text) + if updated_summary is None: + # Track consecutive failures on the same cursor position so we + # don't busy-loop on an unsummarizable exchange every turn. + if exchange_start == self._micro_compact_last_failure_cursor: + self._micro_compact_consecutive_failures += 1 + else: + self._micro_compact_consecutive_failures = 1 + self._micro_compact_last_failure_cursor = exchange_start + + if self._micro_compact_consecutive_failures >= _MICRO_COMPACT_MAX_CONSECUTIVE_FAILURES: + logger.info( + "Micro-compaction: skipping exchange at cursor %d " + "after %d consecutive failures", + exchange_start, self._micro_compact_consecutive_failures, + ) + # Advance the cursor past the stuck exchange so we don't + # retry it every turn. The skipped messages remain in the + # transcript and will be absorbed by the next batch + # compression or defrag. + self._micro_compact_cursor = exchange_end + self._micro_compact_consecutive_failures = 0 + self._micro_compact_last_failure_cursor = -1 + _outcome = "exchange_skipped" + else: + _outcome = "summarize_failed" + self._emit_micro_compaction_telemetry( + outcome=_outcome, + messages_before=_messages_before, + messages_after=len(messages), + tokens_before=_tokens_before, + tokens_after=_tokens_before, + exchange_tokens=_exchange_tokens, + duration_ms=_elapsed_ms(), + ) + return messages + + self._micro_compact_rolling_summary = updated_summary + self._micro_compact_cursor = exchange_end + self._micro_compact_consecutive_failures = 0 + self._micro_compact_last_failure_cursor = -1 + + result = self._splice_micro_compact_result( + messages, exchange_start, exchange_end, supersede=_cumulative, + ) + self._micro_compact_cursor = self._cursor_after_splice(result, exchange_start + 1) + self._sync_micro_compact_to_db(result) + self._emit_micro_compaction_telemetry( + outcome="absorbed", + messages_before=_messages_before, + messages_after=len(result), + tokens_before=_tokens_before, + tokens_after=estimate_messages_tokens_rough(result), + exchange_tokens=_exchange_tokens, + duration_ms=_elapsed_ms(), + ) + return result + + @staticmethod + def _rolling_summary_from_marker(content: Any) -> str: + """Recover the rolling-summary text from a summary marker's content. + + The rolling summary lives in memory, but a resumed session starts with + an empty one while the marker holding every previous exchange is still + in the transcript. Without rehydrating from it, the first post-resume + pass would build a marker from nothing and supersede the one carrying + the whole history. + """ + if not isinstance(content, str) or not content.strip(): + return "" + body = content + # rfind, not find: SUMMARY_PREFIX itself references the heading text, + # so the first occurrence is inside the preamble, not the real heading. + idx = body.rfind(HISTORICAL_TASK_HEADING) + if idx != -1: + body = body[idx + len(HISTORICAL_TASK_HEADING):] + end = body.find(_SUMMARY_END_MARKER) + if end != -1: + body = body[:end] + return body.strip() + + def _cursor_after_splice( + self, + result: List[Dict[str, Any]], + fallback: int, + ) -> int: + """Cursor position just past the summary marker in *result*. + + The cursor must be derived from the spliced list, never carried over + from pre-splice indices. A splice collapses the absorbed span (an + assistant plus its tool results -- often several messages) into a + single marker, and may also drop a superseded marker further back, so + every index after it shifts. Reusing the old ``exchange_end`` left the + cursor pointing into the middle of a *later* exchange's tool group; + the next pass then walked forward to the following assistant and + skipped that exchange entirely, so roughly half the work silently + never happened on tool-bearing conversations. + """ + for idx in range(len(result) - 1, -1, -1): + entry = result[idx] + if isinstance(entry, dict) and entry.get(COMPRESSED_SUMMARY_METADATA_KEY): + return idx + 1 + return fallback + + def _emit_micro_compaction_telemetry( + self, + *, + outcome: str, + messages_before: int, + messages_after: int, + tokens_before: int | None, + tokens_after: int | None, + exchange_tokens: int | None = None, + duration_ms: int | None = None, + ) -> None: + """Emit one content-free JSON log line describing a micro-compaction pass. + + Mirrors ``_emit_compression_attempt_telemetry`` for the batch path. + Message counts move by one or two even when the saving is large, so the + token fields are the ones that actually answer "is this helping?". + ``tokens_delta`` is negative when the pass shrank the transcript, and + the ``*_total`` fields accumulate across the session so a whole run can + be summarised from the last line alone. + """ + try: + delta = None + if tokens_before is not None and tokens_after is not None: + delta = tokens_after - tokens_before + self._micro_compact_tokens_saved_total -= delta + self._micro_compact_passes += 1 + # Cached reads only. The ``threshold_tokens`` / ``context_length`` + # properties resolve lazily and can fire a synchronous /models + # probe on first access (#32221) — telemetry must never be the + # thing that blocks a turn. Unresolved simply reports null. + threshold = self._threshold_tokens + context_limit = self._resolved_context_length + occupancy = None + if threshold and tokens_after is not None and threshold > 0: + occupancy = round(tokens_after / threshold * 100, 1) + payload = { + "event": "micro_compaction", + "session_id": getattr(self, "_session_id", "") or "", + "outcome": outcome, + "messages_before": messages_before, + "messages_after": messages_after, + "tokens_before": _safe_int(tokens_before), + "tokens_after": _safe_int(tokens_after), + "tokens_delta": _safe_int(delta), + "exchange_tokens": _safe_int(exchange_tokens), + "rolling_summary_tokens": estimate_tokens_rough( + self._micro_compact_rolling_summary + ), + "cursor": _safe_int(self._micro_compact_cursor), + "passes_total": self._micro_compact_passes, + "tokens_saved_total": self._micro_compact_tokens_saved_total, + "duration_ms": _safe_int(duration_ms), + # Headroom, not efficiency: how full the window is being kept. + # This is the number that says whether the session can keep + # going without a hard batch compaction. + "threshold_tokens": _safe_int(threshold), + "context_limit": _safe_int(context_limit), + "occupancy_pct": occupancy, + "main_model": self.model or "", + "aux_model": self.summary_model or "", + } + logger.info( + "micro compaction telemetry: %s", + json.dumps(payload, sort_keys=True, separators=(",", ":")), + ) + except Exception as exc: + logger.debug("failed to emit micro-compaction telemetry: %s", exc) + + def _sync_micro_compact_to_db( + self, + compacted_messages: List[Dict[str, Any]], + ) -> None: + """Persist the micro-compacted message set to the session DB. + + Soft-archives every currently-active message row (``active = 0``) + and inserts *compacted_messages* as fresh active rows — atomically, + via ``archive_and_compact``. Then stamps ``_DB_PERSISTED_MARKER`` on + every dict so the upcoming append-only flush (``_persist_session`` → + ``_flush_messages_to_session_db_unlocked``) skips them: they are + already correctly stored. + + Without this, the in-memory-only splice leaves old exchange rows at + ``active=1``, and a session resume double-loads both the summary and + the original messages — blowing past the model's context limit. + """ + session_db = getattr(self, "_session_db", None) + session_id = getattr(self, "_session_id", "") + if not session_db or not session_id: + return + try: + session_db.archive_and_compact(session_id, compacted_messages) + for msg in compacted_messages: + if isinstance(msg, dict): + msg[_DB_PERSISTED_MARKER] = True + except Exception: + logger.info( + "Micro-compaction DB sync failed — resume will double-load " + "compacted messages until the next batch compression" + ) + + def _splice_micro_compact_result( + self, + messages: List[Dict[str, Any]], + splice_start: int, + splice_end: int, + supersede: bool = True, + ) -> List[Dict[str, Any]]: + """Replace *messages[splice_start:splice_end]* with a summary marker. + + The summary marker carries the rolling summary text and the + ``_compressed_summary`` metadata flag so downstream consumers + (resume, handoff, /compress) handle it identically to batch + compaction summaries. + + Alternation safety: the marker is ``assistant``-role. An exchange is + a full agent turn bounded by user messages on both sides (see + ``_find_one_exchange``), so the spliced result is + ``user → marker(assistant) → user`` — valid alternation that the + pre-request ``repair_message_sequence`` pass leaves untouched. A + ``user``-role marker in that position produced ``user → user → user``, + and repair then merged the marker into the neighbouring real user + message: metadata gone, cursor unrecoverable, and the summary text + duplicated into the transcript on every subsequent pass. + + Superseding an earlier marker removes the assistant turn that stood + between two real user messages, leaving them adjacent. Those two are + merged (plain-text only, ``\\n\\n``-joined — the same repair pass 2 + would apply) so the transcript is alternation-valid as returned + rather than relying on downstream repair to fix it up. + """ + summary_text = self._micro_compact_rolling_summary + if not summary_text.strip(): + return messages + + summary_msg = { + "role": "assistant", + "content": self._render_micro_marker_content(summary_text), + COMPRESSED_SUMMARY_METADATA_KEY: True, + # Micro-created marker: eligible for supersede/defrag rewrites. + # Batch markers never carry this key and are never touched — + # their content is not contained in the rolling summary. + MICRO_COMPACT_MARKER_KEY: True, + # Honest provenance (#64650): this marker absorbs only + # assistant/tool content — user turns are never micro-compacted, + # so they remain in the transcript and _transcript_has_real_user_turn + # keeps reporting them directly. + COMPRESSED_SUMMARY_HAS_USER_TURN_KEY: False, + } + + result = messages[:splice_start] + [summary_msg] + messages[splice_end:] + + # The rolling summary is cumulative: this marker already contains + # everything every earlier micro-compaction marker held. Leaving those + # in place stacks near-duplicate copies of the same text — each with + # its own prefix/heading/end-marker scaffolding — so the transcript + # grows with every turn instead of shrinking, which defeats the point. + # Keep only the newest marker. + # Two containment gates before dropping an earlier marker: + # 1. supersede (the rolling summary was non-empty going into this + # pass) — a pass that started from nothing (a resume that could + # not rehydrate) covers one exchange, and dropping the previous + # marker would throw away the entire compacted history. + # 2. MICRO_COMPACT_MARKER_KEY on the candidate — only markers whose + # text is provably inside the rolling summary (created by our own + # splice, or rehydrated into the summary by + # _resolve_compact_cursor) carry it. A batch-compaction marker + # that landed after our last pass holds MORE history than the + # stale rolling summary; dropping it would destroy that history. + if supersede: + marker_idxs = [ + i for i, m in enumerate(result) + if isinstance(m, dict) + and m.get(COMPRESSED_SUMMARY_METADATA_KEY) + and m.get(MICRO_COMPACT_MARKER_KEY) + ] + if len(marker_idxs) > 1: + superseded = set(marker_idxs[:-1]) + result = [m for i, m in enumerate(result) if i not in superseded] + result = self._merge_adjacent_user_turns(result) + + # NOTE: deliberately NO _strip_persistence_markers here. The batch + # path strips because compress() copies head/tail into a rotated + # child session (#57491); micro-compaction archives in place under + # the SAME session id, and the surviving dicts' _db_persisted stamps + # are accurate. Stripping them meant an archive_and_compact failure + # left every previously-persisted message unstamped, and the next + # append-only flush re-inserted them as duplicate active rows on top + # of the still-active originals. _sync_micro_compact_to_db re-stamps + # everything after a SUCCESSFUL archive; on failure the old stamps + # keep the flush idempotent (only the new marker row is appended). + return result + + @staticmethod + def _render_micro_marker_content(summary_text: str) -> str: + """Assemble the marker content wrapper around *summary_text*.""" + return ( + f"{SUMMARY_PREFIX}\n\n" + f"{HISTORICAL_TASK_HEADING}\n" + f"{summary_text.strip()}" + f"\n\n{_SUMMARY_END_MARKER}" + ) + + @staticmethod + def _merge_adjacent_user_turns( + result: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """Merge consecutive plain-text real user turns left by a supersede. + + Dropping a superseded marker removes the assistant turn that separated + two real user messages. Merging them here (``\\n\\n``-joined, exactly + what ``repair_message_sequence`` pass 2 does) keeps every byte the + user typed while restoring alternation deliberately, so the marker + and cursor state are never collateral damage of the downstream repair. + Multimodal (list) content is left alone, mirroring the repair pass. + """ + from agent.turn_context import drop_stale_api_content + + merged: List[Dict[str, Any]] = [] + for msg in result: + prev = merged[-1] if merged else None + if ( + isinstance(msg, dict) + and isinstance(prev, dict) + and msg.get("role") == "user" + and prev.get("role") == "user" + and not msg.get(COMPRESSED_SUMMARY_METADATA_KEY) + and not prev.get(COMPRESSED_SUMMARY_METADATA_KEY) + and isinstance(prev.get("content"), str) + and isinstance(msg.get("content"), str) + ): + prev_content = prev["content"] + new_content = msg["content"] + prev["content"] = ( + (prev_content + "\n\n" + new_content) + if prev_content and new_content + else (prev_content or new_content) + ) + # Merged content invalidates the api_content sidecar (exact + # bytes previously sent for the pre-merge message). + drop_stale_api_content(prev) + continue + merged.append(msg) + return merged + def compress( self, messages: List[Dict[str, Any]], @@ -4807,7 +5945,11 @@ def compress( 1. Prune old tool results (cheap pre-pass, no LLM call) 2. Protect head messages (system prompt + first exchange) 3. Find tail boundary by token budget (~20K tokens of recent context) - 4. Summarize middle turns with structured LLM prompt + 4. Summarize middle turns with structured LLM prompt (skipped + pre-LLM when the middle is below + ``_FEASIBILITY_SKIP_MIDDLE_FRACTION`` of the threshold after a + prior real-usage ineffectiveness strike — the deterministic + fallback drop recovers the negligible savings instead) 5. On re-compression, iteratively update the previous summary Blank platform-echo user rows trailing the latest actionable user @@ -4827,7 +5969,9 @@ def compress( everything else. Inspired by Claude Code's ``/compact``. force: If True, clear any active summary-failure cooldown before running so a manual ``/compress`` can retry immediately after - an auto-compression abort. Auto-compress callers pass False. + an auto-compression abort, and bypass the pre-LLM feasibility + skip so an explicit user request always exercises the full + summary path. Auto-compress callers pass False. memory_context: Optional provider-supplied context to preserve in the summary prompt. Whitespace-only values are ignored. """ @@ -4835,6 +5979,7 @@ def compress( # after compress() returns to decide whether to surface a warning. self._last_summary_dropped_count = 0 self._last_summary_fallback_used = False + self._last_feasibility_skip = False self._last_summary_error = None self._last_aux_model_failure_error = None self._last_aux_model_failure_model = None @@ -4963,6 +6108,7 @@ def compress( # — take the narrow rescan, miss a beyond-window fossil, and discard the # rehydrated state as cross-session leakage (#57835). _previous_summary_before_scan = self._previous_summary + _summary_has_user_turn_before_scan = getattr(self, "_summary_has_user_turn", None) # A persisted handoff summary can sit in the protected head after a # resume (commonly immediately after the system prompt). Search from # the first non-system message through the compression window. On the @@ -5117,12 +6263,73 @@ def _window_row(idx: int, msg: Dict[str, Any]): ) # Phase 3: Generate structured summary - summary_focus_topic = focus_topic or self._derive_auto_focus_topic(messages) - summary = self._generate_summary( - turns_to_summarize, - focus_topic=summary_focus_topic, - memory_context=memory_context, - ) + + # Pre-LLM feasibility check: if the middle section is too small to + # yield meaningful token savings, skip the expensive LLM summarization + # call and fall through to the deterministic message-dropping path + # (which is cheap and always applicable). Without this guard a + # tool-heavy session where the protected tail already holds most of + # the tokens can burn 500+ seconds on a summary call that replaces a + # few lightweight messages, leaving the total token count essentially + # unchanged. + # + # Only fires after at least one prior real-usage ineffectiveness + # strike. The check READS ``_ineffective_compression_count`` but + # never writes it: that strike counter is fed exclusively by real + # provider token counts (see the anti-thrashing verdict in + # _update_token_usage), and consumers latch at >= 2 to disable + # compression entirely. Feasibility skips are tracked separately + # in ``_prellm_skip_count`` for observability. + # + # Skipped when ``force=True`` (manual /compress) so auth/error + # handling paths are always exercised on explicit user request. + feasibility_skip = False + if not force and self._ineffective_compression_count >= 1: + # _record_compression_regions already estimated this exact window + # into the telemetry dict above; reuse it so the log line and + # telemetry can never disagree. The regions helper no-ops when the + # telemetry attr isn't a dict, so fall back to a fresh estimate + # when the key is absent/None (0 is a legitimate value). + middle_tokens = telemetry.get("middle_window_tokens") + if middle_tokens is None: + middle_tokens = estimate_messages_tokens_rough(turns_to_summarize) + if middle_tokens < int( + self.threshold_tokens * _FEASIBILITY_SKIP_MIDDLE_FRACTION + ): + feasibility_skip = True + self._last_feasibility_skip = True + self._prellm_skip_count += 1 + telemetry["prellm_skip_count"] = self._prellm_skip_count + if not self.quiet_mode: + logger.warning( + "Compression: middle section (%d tokens at indices " + "%d-%d) is below %.0f%% of threshold (%d tokens) — " + "skipping LLM summarization, proceeding with " + "deterministic message dropping. prellm_skip_count=%d", + middle_tokens, compress_start, compress_end, + _FEASIBILITY_SKIP_MIDDLE_FRACTION * 100, + self.threshold_tokens, self._prellm_skip_count, + ) + + if feasibility_skip: + summary = None # No LLM call; Phase 4 inserts the deterministic fallback + else: + # Deriving the auto focus topic scans recent user turns — only pay + # for it when a summary will actually be generated. + summary_focus_topic = focus_topic or self._derive_auto_focus_topic(messages) + try: + summary = self._generate_summary( + turns_to_summarize, + focus_topic=summary_focus_topic, + memory_context=memory_context, + ) + except AuxiliaryExplicitCancellation: + # Explicit cancellation is a true no-op. Restore state mutated by + # the resume/handoff self-heal scan before the exception escapes to + # the outer transaction, which restores the transcript and lease. + self._previous_summary = _previous_summary_before_scan + self._summary_has_user_turn = _summary_has_user_turn_before_scan + raise # If summary generation failed, behavior splits on # ``abort_on_summary_failure`` (config: compression.abort_on_summary_failure): @@ -5144,7 +6351,7 @@ def _window_row(idx: int, msg: Dict[str, Any]): # of these cases, rotating into a child session with a placeholder # summary degrades the conversation for zero benefit. Preserve it # unchanged until access is restored or connectivity recovers. - if not summary and ( + if not summary and not feasibility_skip and ( self.abort_on_summary_failure or self._last_summary_auth_failure or self._last_summary_network_failure @@ -5224,15 +6431,26 @@ def _window_row(idx: int, msg: Dict[str, Any]): # content-free "N messages were removed" marker. if not summary: if not self.quiet_mode: - logger.warning("Summary generation failed — inserting deterministic fallback context summary") + if feasibility_skip: + logger.info("Feasibility skip — inserting deterministic fallback context summary") + else: + logger.warning("Summary generation failed — inserting deterministic fallback context summary") n_dropped = compress_end - compress_start self._last_summary_dropped_count = n_dropped self._last_summary_fallback_used = True telemetry["fallback_used"] = True - telemetry["failure_class"] = telemetry.get("failure_class") or "summary_generation_failed" + if feasibility_skip: + # Deliberate optimization, not a summary failure — keep the + # telemetry class distinct so dashboards don't count skips + # as aux-model breakage. + telemetry["failure_class"] = telemetry.get("failure_class") or "feasibility_skip" + else: + telemetry["failure_class"] = telemetry.get("failure_class") or "summary_generation_failed" summary = self._build_static_fallback_summary( turns_to_summarize, - reason=self._last_summary_error, + # A stale error from an earlier real failure must not be + # embedded into a deliberate feasibility skip's fallback. + reason=None if feasibility_skip else self._last_summary_error, ) tail_messages: List[Dict[str, Any]] = [] @@ -5255,8 +6473,45 @@ def _window_row(idx: int, msg: Dict[str, Any]): # last_head_role reads the assembled (post-strip) head; first_tail_role # reads the assembled (post-strip) tail_messages — a stripped stale # handoff must not influence alternation-safe role selection. - last_head_role = compressed[-1].get("role", "user") if compressed else "user" - first_tail_role = tail_messages[0].get("role", "user") if tail_messages else None + # Both are TEMPLATE-VISIBLE roles (``_template_visible_role``), not the + # literal list neighbours: strict Mistral-style templates skip tool + # results and assistant tool-call messages when enforcing + # user/assistant alternation, so the summary must alternate against + # the nearest message the template actually counts. Selecting against + # the literal neighbour (previously ``compressed[-1]``) emitted the + # summary as role="user" behind a ``[user, assistant(tool_calls), + # tool]`` head — which every Mistral-strict backend rejects with a + # Jinja alternation 500, permanently poisoning the session. + last_head_role: Optional[str] = "user" + if compressed: + last_head_role = next( + ( + role + for role in ( + _template_visible_role(m) for m in reversed(compressed) + ) + if role is not None + ), + # Head holds only template-exempt messages: the summary will + # be the first message the template counts, and the sequence + # must open with "user" (handled below alongside the forced + # cases). + None, + ) + first_tail_role = None + first_tail_visible_idx: Optional[int] = None + if tail_messages: + first_tail_visible_idx, first_tail_role = next( + ( + (idx, role) + for idx, role in ( + (idx, _template_visible_role(m)) + for idx, m in enumerate(tail_messages) + ) + if role is not None + ), + (None, None), + ) # When the only protected head message is the system prompt, the # summary becomes the first *visible* message in the API request # (most adapters — Anthropic, Bedrock — send the system prompt as @@ -5282,17 +6537,39 @@ def _window_row(idx: int, msg: Dict[str, Any]): # If no user-role message survives in either the protected head or the # preserved tail, the summary MUST carry role="user" so the request # always has at least one user turn. + # + # A bare role check is not enough: the tail's sole surviving user + # turn can be image-only (a screenshot with no caption). The newest + # image-bearing user message is the ``_strip_historical_media`` + # anchor and is kept byte-for-byte, so it never gains a text + # placeholder — its role is "user" but its text content is empty, + # which backends checking for actual query text still reject. Count + # only user messages with non-empty text as "surviving"; when the + # guard fires, the real (never fabricated) summary text lands in a + # role="user" slot, which is always non-empty (falls back to + # ``_build_static_fallback_summary`` above when generation fails). if not _force_user_leading: + def _is_nonempty_user_turn(message: Dict[str, Any]) -> bool: + return message.get("role") == "user" and bool( + _content_text_for_contains(message.get("content")).strip() + ) + _user_survives = any( - message.get("role") == "user" for message in compressed + _is_nonempty_user_turn(message) for message in compressed ) or any( - message.get("role") == "user" for message in tail_messages + _is_nonempty_user_turn(message) for message in tail_messages ) if not _user_survives: _force_user_leading = True - # Pick a role that avoids consecutive same-role with both neighbors. - # Priority: avoid colliding with head (already committed), then tail. - if last_head_role in {"assistant", "tool"} or _force_user_leading: + # Pick a role that alternates with both template-visible neighbors. + # Priority: alternate against the head (already committed), then tail. + # ``None`` (all-exempt head) means the summary opens the visible + # sequence, which strict templates require to start with "user". + if ( + last_head_role is None + or last_head_role in {"assistant", "tool"} + or _force_user_leading + ): summary_role = "user" else: summary_role = "assistant" @@ -5300,7 +6577,14 @@ def _window_row(idx: int, msg: Dict[str, Any]): # collide with the head, flip it. if first_tail_role is not None and summary_role == first_tail_role: flipped = "assistant" if summary_role == "user" else "user" - if flipped != last_head_role and not _force_user_leading: + # ``last_head_role is None`` (all-exempt head) pins the summary to + # "user" above; flipping to "assistant" would make the visible + # sequence open with "assistant", which strict templates reject. + if ( + flipped != last_head_role + and last_head_role is not None + and not _force_user_leading + ): summary_role = flipped else: # Both roles would create consecutive same-role messages @@ -5329,9 +6613,27 @@ def _window_row(idx: int, msg: Dict[str, Any]): ), }) + # Default merge target: literal tail index 0. For an ordinary + # alternation collision the summary only has to stay *invisible* to + # the template, and a leading template-exempt row (bare tool-call + # assistant message, tool result) is the ideal carrier — it absorbs + # the summary without adding a visible turn, and it leaves the live + # tail user message intact as the model's actual prompt. Retargeting + # to the first template-visible row here would convert that live + # request into the summary carrier for no benefit. + # + # The forced repair path is the exception. There the merge is not + # about alternation but about guaranteeing at least one genuinely + # non-empty role="user" message (an image-only or otherwise + # text-empty surviving user row). An exempt carrier cannot satisfy + # that invariant, so the summary text must land on the + # template-visible row itself. + _merge_target_idx = 0 + if _force_user_leading and first_tail_visible_idx is not None: + _merge_target_idx = first_tail_visible_idx for tail_idx, msg in enumerate(tail_messages): - if _merge_summary_into_tail and tail_idx == 0: - # Merge the summary into the first (post-strip) tail message. + if _merge_summary_into_tail and tail_idx == _merge_target_idx: + # Merge the summary into the tail message that collided. old_content = msg.get("content", "") if _force_user_leading and summary_role == "user": # The summary must be part of the first user-visible @@ -5428,6 +6730,19 @@ def _window_row(idx: int, msg: Dict[str, Any]): _strip_persistence_markers(compressed) self._last_compression_made_progress = True + # Batch compaction invalidates micro-compaction state: the batch + # marker now holds MORE history than the in-memory rolling summary + # (it summarized everything in the window, including exchanges micro + # never absorbed). Keeping the stale summary would let the next micro + # pass supersede-drop or defrag-rewrite content it does not contain. + # Reset instead; the next micro pass rehydrates from the batch marker + # via _resolve_compact_cursor, which re-tags it as micro-eligible + # only after absorbing its content into the rolling summary. + self._micro_compact_rolling_summary = "" + self._micro_compact_cursor = 0 + self._micro_compact_consecutive_failures = 0 + self._micro_compact_last_failure_cursor = -1 + return compressed diff --git a/agent/context_references.py b/agent/context_references.py index eea16ae52b44..ab370a5a5924 100644 --- a/agent/context_references.py +++ b/agent/context_references.py @@ -19,6 +19,7 @@ rf"(?diff|staged)\b|(?Pfile|folder|git|url):(?P{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+))" ) TRAILING_PUNCTUATION = ",.;!?" +_NEEDS_QUOTING = re.compile(r"""[\s()\[\]{}<>"'`]""") _SENSITIVE_HOME_DIRS = (".ssh", ".aws", ".gnupg", ".kube", ".docker", ".azure", ".config/gh") _SENSITIVE_HERMES_DIRS = (Path("skills") / ".hub",) _SENSITIVE_HOME_FILES = ( @@ -60,6 +61,21 @@ class ContextReferenceResult: blocked: bool = False +def format_reference_value(value: str) -> str: + """Quote a reference value so ``REFERENCE_PATTERN`` reads it back whole. + + The unquoted alternative in the pattern is ``\\S+``, so a path containing a + space parses as a truncated ref with the tail left behind as loose text. + Mirrors ``formatRefValue`` in the desktop's directive-text.tsx. + """ + if not _NEEDS_QUOTING.search(value): + return value + for quote in ("`", '"', "'"): + if quote not in value: + return f"{quote}{value}{quote}" + return value + + def parse_context_references(message: str) -> list[ContextReference]: refs: list[ContextReference] = [] if not message: @@ -197,8 +213,12 @@ async def preprocess_context_references_async( f"@ context injection warning: {injected_tokens} tokens exceeds the 25% soft limit ({soft_limit})." ) - stripped = _remove_reference_tokens(message, refs) - final = stripped + # Leave the `@file:`/`@folder:` tokens where the user typed them. The token + # IS the reference, not scaffolding around it: clients render each one as an + # inline chip, so stripping them left a sentence with a hole in it ("review + # and ship") and made the desktop re-derive the refs from the attached block + # to show them as a detached list above the prose. + final = message if warnings: final = f"{final}\n\n--- Context Warnings ---\n" + "\n".join(f"- {warning}" for warning in warnings) if blocks: @@ -308,7 +328,7 @@ def _expand_git_reference( ["git", *args], cwd=cwd, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=30, stdin=subprocess.DEVNULL, **_popen_kwargs, @@ -457,19 +477,6 @@ def _parse_file_reference_value(value: str) -> tuple[str, int | None, int | None return _strip_reference_wrappers(value), None, None -def _remove_reference_tokens(message: str, refs: list[ContextReference]) -> str: - pieces: list[str] = [] - cursor = 0 - for ref in refs: - pieces.append(message[cursor:ref.start]) - cursor = ref.end - pieces.append(message[cursor:]) - text = "".join(pieces) - text = re.sub(r"\s{2,}", " ", text) - text = re.sub(r"\s+([,.;:!?])", r"\1", text) - return text.strip() - - def _is_binary_file(path: Path) -> bool: mime, _ = mimetypes.guess_type(path.name) if mime and not mime.startswith("text/") and not any( @@ -534,7 +541,7 @@ def _rg_files(path: Path, cwd: Path, limit: int) -> list[Path] | None: ["rg", "--files", str(path.relative_to(cwd))], cwd=cwd, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=10, stdin=subprocess.DEVNULL, **_popen_kwargs, diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 59f95be9f291..3256bca9ec00 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -24,10 +24,34 @@ ``run_agent`` keeps thin wrappers for each so existing call sites (``self._compress_context(...)``) keep working. Tests that exercise these paths see no behavioural change. + +Thread-safety contract for extension points (#76354 review) +------------------------------------------------------------ + +When the host-level progress-aware timeout is enabled (the default: +``compression.context_timeout_seconds > 0``), the WHOLE compression pass — +including plugin/legacy **context engines** (``compress()`` / +``on_session_start`` / boundary callbacks) and **memory providers** +(``on_pre_compress`` / ``on_session_switch``) — runs on a pooled daemon +thread, not the conversation thread. Extension authors must assume: + +* Calls may arrive on an arbitrary pooled thread; do not rely on + thread-affinity or ``threading.local`` state shared with the caller. +* The input message list is a private deep snapshot owned by the worker; + engines MAY mutate it in place (legacy contract preserved), and that + mutation is invisible to the live conversation unless the pass commits. +* Publication to caller-visible / durable state happens ONLY on an admitted + commit (:class:`CompressionCommitFence`); after a host timeout the still- + running engine's work is discarded. +* Two compression passes never run concurrently for one session (durable + per-session lock), but passes for DIFFERENT sessions may run concurrently + on pool siblings — engine/provider instances shared across sessions must + be thread-safe or internally locked. """ from __future__ import annotations +import concurrent.futures import copy import inspect import json @@ -40,16 +64,30 @@ import threading from datetime import datetime from pathlib import Path -from typing import Any, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple +from agent.auxiliary_client import AuxiliaryExplicitCancellation from agent.context_engine import ( automatic_compaction_status_message, sanitize_memory_context, ) from agent.model_metadata import estimate_request_tokens_rough +from agent.session_activity import ActivityProvenance, normalize_activity_provenance logger = logging.getLogger(__name__) +# Terminal compression outcomes published by host/hygiene timeout or cooldown +# writers. Detached heartbeat workers must not clobber these back to +# agent.compression after cancel (otherwise timeout is unobservable). Observing +# a terminal stamp (or a cancelled commit fence) also latches the heartbeat +# silent so a later UNKNOWN rewrite cannot re-arm a zombie worker. +_TERMINAL_COMPRESSION_PROVENANCES = frozenset( + { + ActivityProvenance.AGENT_COMPRESSION_TIMEOUT, + ActivityProvenance.AGENT_COMPRESSION_COOLDOWN, + } +) + # Stable marker the gateway matches on to re-tag the auto-compaction lifecycle # status as ``kind="compacting"`` (tui_gateway/server.py::_status_update), so # drivers like the desktop app can show an explicit "Summarizing…" indicator @@ -207,6 +245,202 @@ def _cached_prompt_reflects_builtin_memory(agent: Any, cached_prompt: str) -> bo return True +_COMPRESSOR_ATTEMPT_STATE_FIELDS = ( + "_previous_summary", + "_summary_has_user_turn", + "compression_count", + "_last_compression_savings_pct", + "_ineffective_compression_count", + "_anti_thrash_recovery_deadline", + "_fallback_compression_streak", + "_verify_compaction_cleared_threshold", + "_last_compression_made_progress", + "_summary_failure_cooldown_until", + "_cooldown_persist_failed", + "_last_summary_error", + "_consecutive_timeout_failures", + "_last_summary_dropped_count", + "_last_summary_fallback_used", + "_last_compress_aborted", + "_last_summary_auth_failure", + "_last_summary_network_failure", + "_last_aux_model_failure_error", + "_last_aux_model_failure_model", + "_summary_model_fallen_back", + "summary_model", + "_last_compression_telemetry", + "_active_compression_telemetry", + "_compression_telemetry_seed", +) + +_COMPRESSOR_COOLDOWN_STATE_FIELDS = ( + "_summary_failure_cooldown_until", + "_last_summary_error", + "_cooldown_persist_failed", +) + + +def _snapshot_compressor_attempt_state(compressor: Any) -> dict[str, Any]: + """Copy only mutable bookkeeping owned by one compression attempt. + + The explicit allow-list avoids copying provider clients, SessionDB handles, + locks, and plugin resources. Missing fields are intentionally ignored so + legacy and third-party compressors keep their existing contract. + """ + try: + values = vars(compressor) + except TypeError: + return {} + selected = { + name: values[name] + for name in _COMPRESSOR_ATTEMPT_STATE_FIELDS + if name in values + } + # Copy the collection as one object so aliases between fields (notably + # _active_compression_telemetry and _last_compression_telemetry) survive. + return copy.deepcopy(selected) + + +def _restore_compressor_attempt_state( + compressor: Any, + snapshot: dict[str, Any], + *, + durable_cooldown_authoritative: Optional[bool] = None, + durable_cooldown_state: Optional[dict[str, Any]] = None, +) -> None: + """Restore the safe per-attempt snapshot after a pre-commit hard cancel.""" + # A successful summary clears the durable cooldown before the outer commit + # boundary. Recreate (or clear) that row before restoring exact in-memory + # values, otherwise the next refresh would overwrite this rollback. Unknown + # durable state and intentionally unpersisted local cooldowns are never + # converted into destructive DB writes during cancellation. + if ( + "_summary_failure_cooldown_until" in snapshot + and durable_cooldown_authoritative is not False + and ( + durable_cooldown_authoritative is True + or not bool(snapshot.get("_cooldown_persist_failed", False)) + ) + ): + session_db = vars(compressor).get("_session_db") + session_id = vars(compressor).get("_session_id") + if session_db is not None and session_id: + if durable_cooldown_authoritative is True: + restorer = getattr( + type(session_db), + "restore_compression_failure_cooldown_row", + None, + ) + if not callable(restorer) or durable_cooldown_state is None: + raise RuntimeError( + "exact compression cooldown rollback API is unavailable" + ) + # This API restores raw columns (including expired and null + # combinations), verifies the read-back, and propagates failure. + restorer( + session_db, + session_id, + copy.deepcopy(durable_cooldown_state), + ) + else: + try: + deadline = float( + snapshot["_summary_failure_cooldown_until"] or 0.0 + ) + remaining = max(0.0, deadline - time.monotonic()) + durable_deadline = time.time() + remaining + durable_error = snapshot.get("_last_summary_error") + if remaining > 0: + recorder = getattr( + type(session_db), + "record_compression_failure_cooldown", + None, + ) + if callable(recorder): + recorder( + session_db, + session_id, + durable_deadline, + durable_error, + ) + else: + clearer = getattr( + type(session_db), + "clear_compression_failure_cooldown", + None, + ) + if callable(clearer): + clearer(session_db, session_id) + except Exception: + # Legacy/third-party compatibility path: its existing APIs + # do not provide a verifiable transaction contract. + logger.debug( + "compression cooldown persistence rollback failed", + exc_info=True, + ) + restored = copy.deepcopy(snapshot) + for name, value in restored.items(): + setattr(compressor, name, value) + + +def _capture_authoritative_cooldown_under_lease( + compressor: Any, + attempt_snapshot: dict[str, Any], +) -> tuple[Optional[bool], Optional[dict[str, Any]]]: + """Refresh and snapshot built-in durable cooldown state under the lease. + + Third-party compressors are deliberately not invoked here: arbitrary plugin + callbacks must not run while the session lease is held. A durable read + failure returns ``False`` so rollback cannot mistake unknown durable state + for an authoritative empty row and clear it; an unavailable legacy API + returns ``None`` and preserves the compatibility path. + """ + try: + from agent.context_compressor import ContextCompressor + + if not isinstance(compressor, ContextCompressor): + return None, None + values = vars(compressor) + session_db = values.get("_session_db") + session_id = values.get("_session_id") + raw_reader = ( + getattr( + type(session_db), "get_compression_failure_cooldown_row", None + ) + if session_db is not None + else None + ) + if session_db is None or not session_id: + # Unbound compressors have no durable row to mutate or restore. + return None, None + if not callable(raw_reader): + return False, None + # Capture the exact persisted representation first. The active getter + # intentionally filters expired rows and therefore cannot serve as a + # lossless rollback snapshot. + durable_state = raw_reader(session_db, session_id) + if not isinstance(durable_state, dict): + raise TypeError("raw compression cooldown snapshot must be a mapping") + ContextCompressor.get_active_compression_failure_cooldown( + compressor, + refresh=True, + ) + except Exception as exc: + logger.debug("authoritative compression cooldown capture failed: %s", exc) + return False, None + authoritative = getattr( + compressor, "_last_cooldown_refresh_was_authoritative", None + ) + if authoritative is not True: + return authoritative, None + + values = vars(compressor) + for name in _COMPRESSOR_COOLDOWN_STATE_FIELDS: + if name in values: + attempt_snapshot[name] = copy.deepcopy(values[name]) + return True, copy.deepcopy(durable_state) + + class CompressionCommitFence: """Fence timeout cancellation against post-summary session mutation. @@ -221,8 +455,52 @@ def __init__(self) -> None: self._lock = threading.Lock() self._cancelled = False self._commit_started = False + # Lock-free commit-phase marker (#76354 review F1). ``begin_commit`` + # RETAINS ``self._lock`` until ``finish_commit``, so any host-side + # observation that needs the lock (``try_cancel_before_commit``) + # blocks/space-outs for the whole commit. This Event is set inside + # ``begin_commit`` while the lock is held but is READABLE WITHOUT the + # lock, so a host can observe "a commit was admitted and may be in + # flight" even while the commit itself is hung — which is exactly when + # the overrun warning must be able to fire. + self._commit_phase = threading.Event() + # Lock-free admission revocation (#76354 review F2). Set by + # :meth:`revoke_commit_admission` on ANY host unwind (KeyboardInterrupt, + # cancellation, unexpected exception) without touching the fence lock, + # so a host that cannot afford to block behind an in-flight commit can + # still guarantee no FUTURE commit is admitted. Plain bool store — + # atomic in CPython. + self._admission_revoked = False + # Holder-qualified durable-lock release hook (#76354 review F4; + # transplanted from PR #71569 by @ciabata-git). The worker publishes an + # idempotent, holder-scoped release callable once it owns the durable + # compression lock; a timed-out host invokes it to free the lease + # without racing a NEW holder (DB release is holder-qualified, so a + # stale release can never delete a replacement's row — no ABA). + self._lock_release_guard = threading.Lock() + self._cancelled_lock_release: Optional[Callable[[], None]] = None + self._cancelled_lock_release_requested = False + # Forward-progress telemetry: the compression worker touches this + # whenever the streamed summary call produces a token (see + # ContextCompressor._call_summary_llm). Waiters use it to distinguish + # a SLOW-but-alive summary model from a HUNG one, so slow models are + # not killed by a fixed wall-clock deadline while tokens are moving. + self._last_progress = time.monotonic() + + def touch_progress(self) -> None: + """Record forward progress (e.g. a streamed summary token arriving). + + Called from the compression worker thread; read by async waiters via + :meth:`seconds_since_progress`. A bare float store is atomic in + CPython, so no lock is needed. + """ + self._last_progress = time.monotonic() + + def seconds_since_progress(self) -> float: + """Seconds since the worker last reported forward progress.""" + return max(0.0, time.monotonic() - self._last_progress) - def cancel_before_commit(self) -> bool: + def cancel_before_commit(self, cancel_event: Any = None) -> bool: """Cancel a pending commit, or wait for an active commit to finish. Returns ``True`` when cancellation won before the commit boundary. @@ -231,8 +509,12 @@ def cancel_before_commit(self) -> bool: """ with self._lock: if self._commit_started: + if cancel_event is not None: + cancel_event.set() return False self._cancelled = True + if cancel_event is not None: + cancel_event.set() return True def try_cancel_before_commit(self) -> Optional[bool]: @@ -251,19 +533,559 @@ def try_cancel_before_commit(self) -> Optional[bool]: finally: self._lock.release() - def begin_commit(self) -> bool: - """Enter the commit boundary unless cancellation already won.""" + def begin_commit(self, cancel_event: Any = None) -> bool: + """Atomically admit commit unless a hard cancellation already won.""" self._lock.acquire() - if self._cancelled: + if ( + self._cancelled + or self._admission_revoked + or (cancel_event is not None and bool(cancel_event.is_set())) + ): + self._cancelled = True self._lock.release() + if self._admission_revoked: + # Round-2 #1: a revoke that lost the fence-lock race to this + # very begin_commit deferred its lease release; the commit was + # refused, so the release is safe (and idempotent with the + # worker's own holder-qualified cleanup) right now. + self.release_cancelled_compression_lock() return False self._commit_started = True + # Set while the fence lock is held so observers can never see + # commit_in_flight=True for a commit that lost to cancellation. + self._commit_phase.set() return True def finish_commit(self) -> None: """Leave a commit boundary entered by :meth:`begin_commit`.""" + self._commit_phase.clear() + self._lock.release() + if self._admission_revoked: + # Round-2 #1: a revoke that arrived while THIS commit was in + # flight deferred its durable-lease release rather than freeing + # the lock out from under an active SessionDB mutation. The + # commit is now fully complete, so perform the deferred release + # here — promptly, without relying on the (possibly parked) + # worker thread's outer cleanup. Idempotent with that cleanup: + # the DB release is holder-qualified. + self.release_cancelled_compression_lock() + + @property + def commit_in_flight(self) -> bool: + """Lock-free read: an admitted commit has begun and not yet finished. + + Safe to call from the host while the worker holds the fence lock for + the whole commit (a hung SessionDB write). Hosts use this to reach + their overrun-warning loop WHILE the commit is blocked instead of + spinning on ``try_cancel_before_commit`` (which needs the lock the + worker retains until ``finish_commit``). + """ + return self._commit_phase.is_set() + + @property + def is_cancelled(self) -> bool: + """True after cancellation won before the commit boundary.""" + return self._cancelled or self._admission_revoked + + def revoke_commit_admission(self) -> None: + """Revoke FUTURE commit admission without blocking on the fence lock. + + #76354 review F2: every host unwind path (KeyboardInterrupt, task + cancellation, unexpected exception while waiting) must guarantee a + detached worker cannot later enter the commit boundary and mutate + durable/session state. The flag store is lock-free: a commit that is + ALREADY in flight cannot be safely abandoned (the invariant "commit + never abandoned mid-mutation" holds), but no NEW commit will be + admitted after this call — ``begin_commit`` re-checks the flag under + the fence lock. + + Round-2 #1 (durable-lease timing): the worker's holder-qualified + lease release (F4) must NOT run while an admitted commit is still + mutating SessionDB — a second compressor could otherwise acquire the + durable lock mid-commit and interleave with the first commit's + writes. The release decision is therefore made under the fence lock: + + - non-blocking acquire succeeds → no commit is in flight (an + admitted commit RETAINS the lock until ``finish_commit``), so the + lease is released immediately, while still holding the lock so a + concurrent ``begin_commit`` cannot slip in between the check and + the release (it would be refused anyway — the flag is already set). + - acquire fails → the lock holder is either an in-flight commit or a + transient boundary (lock-setup / cancel admission). Defer: the + release then runs in ``finish_commit`` (after the mutation fully + completes) or on the ``begin_commit``-refusal path, whichever the + worker reaches first. Both are idempotent with the worker's own + outer cleanup because the DB release is holder-qualified. + """ + self._admission_revoked = True + if self._lock.acquire(blocking=False): + try: + self.release_cancelled_compression_lock() + finally: + self._lock.release() + # else: deferred — finish_commit()/begin_commit() re-check + # _admission_revoked and perform the release once no commit can be + # mid-mutation. + + # ── Holder-qualified durable-lease cancellation (#76354 F4) ────────── + # Transplanted from PR #71569 (@ciabata-git): the worker publishes an + # idempotent, holder-scoped release hook once it owns the durable + # compression lock, and the host invokes it after winning cancellation. + # ABA safety comes from SessionDB.release_compression_lock being + # holder-qualified (DELETE ... WHERE holder = ?), so a stale release can + # never free a NEW holder's lease. + + def begin_lock_setup(self) -> bool: + """Fence durable-lock acquisition and release-hook publication. + + The caller keeps the fence until it has either published the exact + holder-qualified release hook or established that no lock was + acquired. A timeout cannot therefore win in the gap between acquiring + the durable lock and making its cancellation cleanup callable. + """ + self._lock.acquire() + if self._cancelled or self._admission_revoked: + self._lock.release() + return False + return True + + def finish_lock_setup(self) -> None: + """Leave a lock setup boundary entered by :meth:`begin_lock_setup`.""" self._lock.release() + def register_cancelled_lock_release( + self, release: Callable[[], None] + ) -> bool: + """Publish the timed-out worker's holder-qualified lock release. + + Returns whether cancellation cleanup was requested before publication. + In that race, the release runs synchronously before this method returns. + """ + with self._lock_release_guard: + self._cancelled_lock_release = release + requested = self._cancelled_lock_release_requested + if requested: + release() + return requested + + def clear_cancelled_lock_release(self, release: Callable[[], None]) -> None: + """Forget ``release`` after the worker's normal cleanup finishes.""" + with self._lock_release_guard: + if self._cancelled_lock_release is release: + self._cancelled_lock_release = None + + def release_cancelled_compression_lock(self) -> None: + """Release the cancelled worker's lock without finalizing its clients. + + Callers invoke this only after cancellation won (fence cancelled or + admission revoked). A request that races ahead of lock-hook + publication is retained and fulfilled synchronously when the worker + publishes the hook. + """ + with self._lock_release_guard: + self._cancelled_lock_release_requested = True + release = self._cancelled_lock_release + if release is not None: + release() + + +# Defaults for the in-agent (non-hygiene) progress-aware compress_context wrap. +# Mirror hermes_cli.config.DEFAULT_CONFIG["compression"] keys of the same name. +DEFAULT_CONTEXT_TIMEOUT_SECONDS = 120.0 +DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS = 600.0 + +# Shared daemon pool for sync compress_context timeout wraps — analogous to +# asyncio's default executor used by gateway session hygiene's +# ``loop.run_in_executor(None, ...)``, but daemon so a fence-cancelled hung +# worker cannot block interpreter exit via concurrent.futures' atexit join. +# Created lazily; never shut down per call (a timed-out worker may still be +# winding down after fence cancel). +_compress_timeout_executor = None +_compress_timeout_executor_lock = threading.Lock() + +# Commit-phase overrun wait slice: once an in-flight SessionDB commit runs +# past the total ceiling, keep waiting in bounded increments of this size so +# every overrun window produces a fresh (escalating) log line instead of one +# silent unbounded future.result(). Clamped down to the ceiling for tiny test +# ceilings so overrun reporting stays observable at test timescales. +_COMMIT_OVERRUN_WAIT_SLICE_SECONDS = 30.0 + +# Bounded admission for the shared compress-timeout pool (#76354 review F6). +# The stdlib executor queue is unbounded: with all four workers wedged in hung +# summaries, a fifth compression would queue silently, wait out its whole +# timeout without ever starting, and remain eligible to run as a stale job +# whenever a worker recovered. Admission is therefore capped at the worker +# count — when every worker slot is occupied (running OR admitted-not-started) +# submission FAILS FAST and the caller continues without compression. +# +# Recovery contract when all workers are wedged: new compressions fail fast +# (no queue growth, conversation continues uncompressed, a warning is logged +# each attempt); wedged workers are fence-cancelled so they cannot publish +# anything when they eventually return, and each recovery frees its admission +# slot via the future done-callback, restoring normal service. If a worker +# NEVER returns, its slot is lost for the process lifetime — bounded, +# observable degradation instead of an unbounded stale-job queue. +_COMPRESS_EXECUTOR_MAX_WORKERS = 4 +_compress_admission_lock = threading.Lock() +_compress_admitted_count = 0 + + +class CompressionExecutorSaturatedError(RuntimeError): + """All compression pool slots are occupied; submission was refused.""" + + +def _try_admit_compression_job() -> bool: + """Reserve one bounded compression-pool admission slot (F6).""" + global _compress_admitted_count + with _compress_admission_lock: + if _compress_admitted_count >= _COMPRESS_EXECUTOR_MAX_WORKERS: + return False + _compress_admitted_count += 1 + return True + + +def _release_compression_admission(_future=None) -> None: + """Free an admission slot (future done-callback or failed submit).""" + global _compress_admitted_count + with _compress_admission_lock: + if _compress_admitted_count > 0: + _compress_admitted_count -= 1 + + +def _get_compress_timeout_executor(): + """Return the process-wide compress-timeout DaemonThreadPoolExecutor.""" + global _compress_timeout_executor + executor = _compress_timeout_executor + if executor is not None: + return executor + from tools.daemon_pool import DaemonThreadPoolExecutor + + with _compress_timeout_executor_lock: + if _compress_timeout_executor is None: + # Small pool: compress is rare and heavy. Sized for a few + # overlapping calls (live compress + fence-cancelled workers + # still winding down), not asyncio's min(32, cpu+4) fan-out. + _compress_timeout_executor = DaemonThreadPoolExecutor( + max_workers=_COMPRESS_EXECUTOR_MAX_WORKERS, + thread_name_prefix="compress-ctx-timeout", + ) + return _compress_timeout_executor + + +def resolve_context_compression_timeouts( + compression_cfg: Optional[dict] = None, +) -> Tuple[float, float]: + """Return ``(idle_timeout_seconds, total_ceiling_seconds)``. + + ``idle_timeout_seconds <= 0`` disables the owned progress-aware wrapper. + The ceiling is clamped to at least one idle window when the idle budget + is positive, matching gateway hygiene semantics. + """ + idle = DEFAULT_CONTEXT_TIMEOUT_SECONDS + ceiling = DEFAULT_CONTEXT_TOTAL_CEILING_SECONDS + cfg = compression_cfg + if cfg is None: + try: + from hermes_cli.config import load_config + + raw = load_config() + maybe = raw.get("compression", {}) if isinstance(raw, dict) else {} + cfg = maybe if isinstance(maybe, dict) else {} + except Exception: + cfg = {} + if isinstance(cfg, dict): + raw_idle = cfg.get("context_timeout_seconds") + if raw_idle is not None: + try: + parsed = float(raw_idle) + # Explicit 0/negative disables; positive values win. + idle = parsed + except (TypeError, ValueError): + pass + raw_ceiling = cfg.get("context_total_ceiling_seconds") + if raw_ceiling is not None: + try: + parsed = float(raw_ceiling) + if parsed > 0: + ceiling = parsed + except (TypeError, ValueError): + pass + if idle > 0: + ceiling = max(ceiling, idle) + return idle, ceiling + + +def run_compress_context_with_progress_timeout( + *, + worker: Callable[[CompressionCommitFence], Tuple[list, str]], + messages: list, + system_prompt_fallback: Any, + idle_timeout_seconds: float, + total_ceiling_seconds: float, + on_timeout: Optional[Callable[[float, float, float], None]] = None, + on_commit_overrun: Optional[Callable[[float, float], None]] = None, + fence: Optional[CompressionCommitFence] = None, + telemetry_agent: Any = None, +) -> Tuple[list, str]: + """Run ``worker(fence)`` under a sync progress-aware timeout. + + The idle budget is inactivity-based (same idea as gateway session hygiene): + streamed summary progress via :meth:`CompressionCommitFence.touch_progress` + extends the wait. A hard ceiling still bounds a degenerate trickle stream. + + When cancellation wins before the commit boundary, returns + ``(messages, system_prompt_fallback)`` immediately and leaves the worker + thread detached — the fence prevents a late commit from mutating session + state. When the worker already entered the commit boundary, waits for that + commit to finish and returns its result. + + Timeout budgets (``idle_timeout_seconds`` / ``total_ceiling_seconds``) cover + the **pre-commit** wait only — the summary / stream phase before + :meth:`CompressionCommitFence.begin_commit`. Once the worker holds the + commit fence, SessionDB mutation is already in flight and cannot be safely + abandoned without risking transcript divergence; the commit is therefore + always allowed to complete. The commit-phase wait is still *bounded in + increments* against the remaining total ceiling: if the commit runs past + ``total_ceiling_seconds``, the overrun is logged loudly (escalating from + WARNING to ERROR on repeat) and surfaced once via ``on_commit_overrun``, + while the host keeps waiting in bounded slices until the commit finishes. + The documented guarantee is: **summary phase bounded by the ceiling; + commit phase logged + surfaced if it exceeds it** (never silently hung, + never abandoned mid-commit). + + ``system_prompt_fallback`` may be a string or a zero-arg callable resolved + only on the timeout path, so successful compression never pays for (or + fails on) an eager prompt rebuild. + """ + if idle_timeout_seconds <= 0: + raise ValueError( + "run_compress_context_with_progress_timeout requires " + "idle_timeout_seconds > 0; call compress_context directly to disable" + ) + + def _resolve_fallback_prompt() -> str: + if callable(system_prompt_fallback): + return system_prompt_fallback() + return system_prompt_fallback + + fence = fence if fence is not None else CompressionCommitFence() + ceiling = max(float(total_ceiling_seconds), float(idle_timeout_seconds)) + idle = float(idle_timeout_seconds) + # Sync mirror of gateway session-hygiene's run_in_executor(None, ...) + + # wait_for loop (gateway/run.py): offload compress_context onto the shared + # daemon pool, poll with an inactivity budget + total ceiling, then + # fence-cancel on timeout so a late commit cannot land. Daemon workers + # match tool_executor: a cancelled hung summary must not block process exit. + from tools.thread_context import propagate_context_to_thread + + executor = _get_compress_timeout_executor() + # Bounded admission (#76354 F6): refuse rather than queue when every pool + # slot is occupied. A queued job would silently wait out its whole budget + # without starting and stay eligible to run as a stale cancelled job when + # a worker recovers. Fail fast: continue without compression this cycle. + if not _try_admit_compression_job(): + logger.warning( + "Context compression pool saturated (%d workers busy) — " + "refusing new compression this cycle and continuing without " + "compression. Wedged workers are fence-cancelled and free their " + "slot when they return; if this persists, check the summary " + "provider health.", + _COMPRESS_EXECUTOR_MAX_WORKERS, + ) + # Round-2 #6: saturation refusals must be visible in the same + # telemetry stream as every other failed attempt, or a wedged pool + # looks like compression simply stopped being attempted. + if telemetry_agent is not None: + _emit_compression_attempt_telemetry( + telemetry_agent, + started_at=time.monotonic(), + commit_status="aborted", + split_status="aborted", + failure_class="pool_saturated", + ) + return messages, _resolve_fallback_prompt() + + def _fence_gated_worker(worker_fence: CompressionCommitFence): + # F6: an admitted job can still start after the host stopped waiting + # (worker slot freed late). Check the fence BEFORE any expensive + # summary work so a stale job never burns an LLM call; its return + # value is discarded by the already-departed host. + if worker_fence.is_cancelled: + logger.info( + "Skipping stale compression job: fence cancelled before start" + ) + return messages, "" + return worker(worker_fence) + + # Bare pool workers start with an empty ContextVar map; propagate the + # parent conversation/approval context into the worker. + try: + future = executor.submit( + propagate_context_to_thread(_fence_gated_worker), fence + ) + except BaseException: + _release_compression_admission() + raise + future.add_done_callback(_release_compression_admission) + wait_started = time.monotonic() + # F2: EVERY host unwind (KeyboardInterrupt, task cancellation, unexpected + # exception while waiting) must revoke future commit admission before the + # host resumes, or a detached worker could later commit and mutate durable + # state behind the caller's back. ``handled_exit`` marks the paths that + # settle admission themselves (worker result returned, or fence cancel + # won); everything else revokes in the ``finally``. + handled_exit = False + try: + while True: + waited = time.monotonic() - wait_started + remaining_ceiling = ceiling - waited + if remaining_ceiling <= 0: + break + # #76354 S3 analogue for this wait: charge the idle budget from + # the LAST PROGRESS event, not from the start of this wait slice. + # Waiting a full ``idle`` after progress that landed early in the + # previous slice would allow silence to approach 2x the budget. + since_progress = fence.seconds_since_progress() + wait_slice = min( + max(idle - since_progress, 0.005), remaining_ceiling + ) + try: + result = future.result(timeout=wait_slice) + handled_exit = True + return result + except concurrent.futures.TimeoutError: + waited = time.monotonic() - wait_started + since_progress = fence.seconds_since_progress() + if since_progress < idle and waited < ceiling: + logger.info( + "Context compression still streaming after %.0fs " + "(last progress %.1fs ago) — extending wait " + "(ceiling %.0fs)", + waited, + since_progress, + ceiling, + ) + continue + break + + # F6: a not-yet-started future must not linger as a stale queued job. + # cancel() is a no-op for a running worker (fence handles that path). + future.cancel() + + cancelled: Optional[bool] = None + while cancelled is None: + # F1: ``begin_commit`` retains the fence lock until + # ``finish_commit``, so a hung commit makes + # ``try_cancel_before_commit`` return None forever. The lock-free + # phase marker breaks the spin so the overrun-warning loop below + # is reachable WHILE the commit is still blocked. + if fence.commit_in_flight: + cancelled = False + break + cancelled = fence.try_cancel_before_commit() + if cancelled is None: + # Round-2 #5: the fence is only held transiently here (lock + # setup / cancel admission — an in-flight commit is caught by + # the commit_in_flight check above), but that window rides + # SessionDB write patience and can last seconds. 25ms keeps + # sub-tick latency without a 1kHz spin. + time.sleep(0.025) + if not cancelled: + # Pre-commit ceiling already elapsed, but begin_commit() won the + # race. Waiting is intentional: SessionDB mutation cannot be + # fence-cancelled. The wait is bounded in increments against the + # remaining ceiling: a commit that overruns total_ceiling_seconds + # is logged loudly and surfaced once (on_commit_overrun), then + # waited on in bounded slices with escalating log level until it + # completes. Guarantee: summary phase bounded by ceiling; commit + # phase logged + surfaced if it exceeds it — never silently hung, + # never abandoned mid-commit. F1: this loop is reachable WHILE + # the commit is blocked (commit_in_flight is lock-free), so the + # warning + on_commit_overrun fire during the hang, not after it. + overrun_surfaced = False + overrun_reports = 0 + while True: + waited = time.monotonic() - wait_started + remaining = ceiling - waited + if remaining <= 0: + # Ceiling breached while the commit is in flight. Wait in + # bounded increments so each overrun window is visible in + # logs rather than one silent unbounded block. + remaining = min( + _COMMIT_OVERRUN_WAIT_SLICE_SECONDS, + max(ceiling, 0.05), + ) + overrun_reports += 1 + log = ( + logger.warning if overrun_reports <= 2 else logger.error + ) + log( + "Context compression SessionDB commit still running " + "%.1fs past the total ceiling (waited %.1fs, ceiling " + "%.1fs); commit cannot be abandoned mid-flight — " + "continuing to wait (check SessionDB health if this " + "persists)", + waited - ceiling, + waited, + ceiling, + ) + if not overrun_surfaced and on_commit_overrun is not None: + overrun_surfaced = True + try: + on_commit_overrun(waited, ceiling) + except Exception: + logger.debug( + "compress_context commit-overrun callback " + "failed", + exc_info=True, + ) + try: + result = future.result(timeout=remaining) + handled_exit = True + return result + except concurrent.futures.TimeoutError: + # Fence progress (commit-phase touch_progress) is + # informative only — the commit must complete regardless; + # loop and re-report with the updated overrun window. + continue + + # Idle-timeout path: cancellation won before the commit boundary. + # The fence already blocks any future commit; F4 additionally frees + # the timed-out worker's durable lease via the holder-qualified hook + # so a NEW compressor can acquire the lock immediately (no ABA: the + # DB release is holder-scoped). + handled_exit = True + fence.release_cancelled_compression_lock() + waited = time.monotonic() - wait_started + since_progress = fence.seconds_since_progress() + if on_timeout is not None: + try: + on_timeout(idle, waited, since_progress) + except Exception: + logger.debug( + "compress_context timeout callback failed", + exc_info=True, + ) + else: + logger.warning( + "Context compression made no progress for %.1fs " + "(total wait %.1fs, ceiling %.1fs); continuing without " + "compression", + since_progress, + waited, + ceiling, + ) + # Leave the future on the shared pool: fence cancel won, so a late + # commit cannot land (same detachment model as gateway hygiene). + return messages, _resolve_fallback_prompt() + finally: + if not handled_exit: + # F2: KeyboardInterrupt / cancellation / any unexpected exception + # while waiting — revoke commit admission (and release the + # worker's durable lease via the holder-qualified hook) before + # the host unwinds, so the detached worker can never publish. + fence.revoke_commit_admission() + def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool: """Whether the live in-memory SessionDB class structurally predates locks. @@ -288,13 +1110,21 @@ def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool: return False -def _refresh_persisted_compression_guards(compressor: Any) -> None: +def _refresh_persisted_compression_guards( + compressor: Any, + *, + include_cooldown: bool = True, +) -> None: """Refresh durable automatic-compression guards on a built-in compressor.""" - method_calls = ( - ("get_active_compression_failure_cooldown", {"refresh": True}), + method_calls = [ ("_load_fallback_compression_streak", {}), ("_load_ineffective_compression_count", {}), - ) + ] + if include_cooldown: + method_calls.insert( + 0, + ("get_active_compression_failure_cooldown", {"refresh": True}), + ) for method_name, kwargs in method_calls: method = getattr(type(compressor), method_name, None) if not callable(method): @@ -373,6 +1203,125 @@ def compression_skipped_due_to_lock(agent: Any) -> bool: return _sig is True or isinstance(_sig, str) +def _adopt_live_compression_child( + agent: Any, + session_db: Any, + parent_session_id: str, +) -> Optional[List[Dict[str, Any]]]: + """Move a stale compression contender onto the unique durable child. + + Resolve and load first, then mutate the live agent. This ordering keeps the + stale contender fail-closed when lineage is ambiguous or the compacted + handoff cannot be read. + """ + finder = getattr(type(session_db), "find_live_compression_child", None) + loader = getattr(type(session_db), "get_messages_as_conversation", None) + if not callable(finder) or not callable(loader): + return None + child = finder(session_db, parent_session_id) + if not child or not child.get("id"): + return None + child_session_id = str(child["id"]) + recovered = loader(session_db, child_session_id) + if not isinstance(recovered, list) or not recovered: + return None + # Revalidate after loading: the child may have rotated or a competing + # continuation may have appeared between the two DB reads. + confirmed = finder(session_db, parent_session_id) + if not confirmed or str(confirmed.get("id") or "") != child_session_id: + return None + + agent.session_id = child_session_id + try: + from gateway.session_context import set_current_session_id + + set_current_session_id(child_session_id) + except Exception: + os.environ["HERMES_SESSION_ID"] = child_session_id + try: + from hermes_logging import set_session_context + + set_session_context(child_session_id) + except Exception: + pass + + agent._session_db_created = True + if child.get("system_prompt"): + agent._cached_system_prompt = child["system_prompt"] + agent._last_flushed_db_idx = len(recovered) + agent._flushed_db_message_session_id = child_session_id + agent._flushed_db_message_ids = { + id(message) for message in recovered if isinstance(message, dict) + } + + on_session_start = getattr(agent.context_compressor, "on_session_start", None) + if callable(on_session_start): + try: + on_session_start( + child_session_id, + boundary_reason="compression", + old_session_id=parent_session_id, + session_db=session_db, + platform=getattr(agent, "platform", None) or "cli", + conversation_id=getattr(agent, "_gateway_session_key", None), + ) + except Exception as exc: + logger.debug("context engine compression-child adoption failed: %s", exc) + else: + bind_state = getattr(agent.context_compressor, "bind_session_state", None) + if callable(bind_state): + try: + bind_state(session_db=session_db, session_id=child_session_id) + except Exception: + pass + try: + if agent._memory_manager: + agent._memory_manager.on_session_switch( + child_session_id, + parent_session_id=parent_session_id, + reset=False, + reason="compression", + ) + except Exception as exc: + logger.debug("memory manager compression-child adoption failed: %s", exc) + + return recovered + + +def recover_rotated_compression_session( + agent: Any, +) -> Optional[List[Dict[str, Any]]]: + """Recover a stale live agent before a new turn writes to its old parent.""" + session_db = getattr(agent, "_session_db", None) + session_id = getattr(agent, "session_id", None) or "" + if session_db is None or not session_id: + return None + try: + if not _session_was_rotated_by_compression(session_db, session_id): + return None + # Rotation publication holds the parent compression lease until the + # child handoff is durable. A concurrent turn waits briefly rather than + # observing the intentional parent-ended/child-empty intermediate state. + holder_getter = getattr(session_db, "get_compression_lock_holder", None) + for attempt in range(21): + recovered = _adopt_live_compression_child(agent, session_db, session_id) + if recovered is not None: + return recovered + holder = holder_getter(session_id) if callable(holder_getter) else None + if not holder or attempt == 20: + return None + time.sleep(0.05) + return None + except Exception as exc: + logger.warning( + "compression session recovery failed for session=%s (%s: %s)", + session_id, + type(exc).__name__, + exc, + ) + return None + + def _compression_lock_holder(agent: Any) -> str: """Build a unique holder id for the lock: pid:tid:agent-instance:uuid. @@ -435,8 +1384,17 @@ def _supported_compression_kwargs( class _CompressionActivityHeartbeat: """Refresh the agent inactivity tracker while compression blocks in an aux call.""" - def __init__(self, agent: Any, interval_seconds: float | None = None) -> None: + def __init__( + self, + agent: Any, + interval_seconds: float | None = None, + commit_fence: Optional[CompressionCommitFence] = None, + ) -> None: self._agent = agent + self._commit_fence = commit_fence + # Latched once host cancel/timeout wins or a terminal stamp is observed, + # so a later UNKNOWN rewrite cannot re-arm a detached zombie heartbeat. + self._suppressed = False if interval_seconds is None: interval_seconds = getattr(agent, "_compression_activity_heartbeat_interval", 60.0) try: @@ -454,7 +1412,10 @@ def __init__(self, agent: Any, interval_seconds: float | None = None) -> None: ) def start(self) -> "_CompressionActivityHeartbeat": - self._touch("context compression started") + # A new compression episode always republishes agent.compression even + # if a prior timeout/cooldown stamp is still on the agent. + self._suppressed = False + self._touch("context compression started", allow_terminal_overwrite=True) self._thread.start() return self @@ -462,18 +1423,63 @@ def stop(self, desc: str = "context compression completed") -> None: self._stop.set() if self._thread.is_alive() and threading.current_thread() is not self._thread: self._thread.join(timeout=1.0) - self._touch(desc) + # Host timeout already owns the terminal stamp; a detached worker's + # late stop must not republish agent.compression / "completed". + if self._should_suppress(): + return + # Terminal completed/failed must reach SessionDB even inside the + # ordinary 60s activity persist window — otherwise durable labels + # stay on "context compression in progress" after /compress (which + # never hits run_conversation's turn-end clear). + self._touch(desc, force_persist=True) + + def _fence_cancelled(self) -> bool: + fence = self._commit_fence + return fence is not None and fence.is_cancelled + + def _should_suppress(self) -> bool: + if self._suppressed: + return True + if self._fence_cancelled(): + self._suppressed = True + return True + return False - def _touch(self, desc: str) -> None: + def _touch( + self, + desc: str, + *, + allow_terminal_overwrite: bool = False, + force_persist: bool = False, + ) -> None: try: + if not allow_terminal_overwrite: + if self._should_suppress(): + return + current = normalize_activity_provenance( + getattr(self._agent, "_last_activity_provenance", None) + ) + if current in _TERMINAL_COMPRESSION_PROVENANCES: + self._suppressed = True + return touch = getattr(self._agent, "_touch_activity", None) if callable(touch): - touch(desc) + # Re-check after reading provenance: host may cancel/stamp + # TIMEOUT between the earlier guard and the write. + if not allow_terminal_overwrite and self._should_suppress(): + return + touch( + desc, + provenance=ActivityProvenance.AGENT_COMPRESSION, + force_persist=force_persist, + ) except Exception: logger.debug("compression activity heartbeat touch failed", exc_info=True) def _run(self) -> None: while not self._stop.wait(self._interval_seconds): + if self._should_suppress(): + return self._touch("context compression in progress") @@ -533,7 +1539,17 @@ def _run(self) -> None: # by the TTL the acquirer set — the lock can never be held past its TTL # by a stuck refresher. consecutive_failures = 0 - while not self._stop.wait(self._refresh_interval_seconds): + # First refresh happens immediately, not one interval late. Everything + # between try_acquire() and start() (the rotation-ownership lookup, the + # durable-breaker re-read, thread startup) is charged against the very + # first lease, so on a short TTL under load the lock could already be + # expired — and reclaimable by a competing path — before tick #1. + first = True + while first or not self._stop.wait(self._refresh_interval_seconds): + if first: + first = False + if self._stop.is_set(): + break try: refreshed = self._db.refresh_compression_lock( self._session_id, @@ -892,6 +1908,7 @@ def _message_text(message: Any) -> str: "_empty_recovery_synthetic", "_verification_stop_synthetic", "_pre_verify_synthetic", + "_dropped_toolcall_nudge", ) @@ -1149,6 +2166,11 @@ def compress_context( prompt — the session is NOT rotated. Callers should detect the no-op via ``len(returned) == len(input)`` and stop the retry loop. """ + _compressor_attempt_snapshot = _snapshot_compressor_attempt_state( + agent.context_compressor + ) + _durable_cooldown_authoritative: Optional[bool] = None + _durable_cooldown_state: Optional[dict[str, Any]] = None if ( defer_context_engine_notification and callable(getattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)) @@ -1194,8 +2216,13 @@ def compress_context( if getattr(agent, "api_mode", None) == "codex_app_server": _codex_fence_entered = False if commit_fence is not None: - _codex_fence_entered = commit_fence.begin_commit() + _codex_fence_entered = commit_fence.begin_commit( + getattr(agent, "_hard_interrupt_requested", None) + ) if not _codex_fence_entered: + _restore_compressor_attempt_state( + agent.context_compressor, _compressor_attempt_snapshot + ) existing_prompt = getattr(agent, "_cached_system_prompt", None) if not existing_prompt: existing_prompt = agent._build_system_prompt(system_message) @@ -1252,8 +2279,11 @@ def compress_context( # parent_session_id child, no # `name #N` renumber, no contextvar/env/logging re-sync, no memory/context- # engine session-switch. The conversation keeps one durable id for life, - # eliminating the session-rotation bug cluster. Default False during rollout. - in_place = bool(getattr(agent, "compression_in_place", False)) + # eliminating the session-rotation bug cluster. Default True (2107b86024). + # Default True matches DEFAULT_CONFIG / #38763. A missing attribute must + # NOT fall back to rotation mode — that re-enables the pre-lease drift + # path and can wedge busy sessions that never set the flag. + in_place = bool(getattr(agent, "compression_in_place", True)) # Set True once the in-place DB write actually completes (the DB block can # raise and skip it). Surfaced to the gateway via agent._last_compaction_in_place. compacted_in_place = False @@ -1353,6 +2383,19 @@ def _complete_compaction_lifecycle() -> None: _lock_ttl = 300.0 _lock_refresh_interval = getattr(agent, "_compression_lock_refresh_interval", None) _lock_refresher: Optional[_CompressionLockLeaseRefresher] = None + # F4 (#76354, transplanted from PR #71569 by @ciabata-git): fence the + # durable-lock acquisition + release-hook publication so a host timeout + # can never win in the gap between acquiring the durable lock and having + # a holder-qualified way to release it. + _lock_setup_entered = False + + def _finish_lock_setup() -> None: + nonlocal _lock_setup_entered + if not _lock_setup_entered or commit_fence is None: + return + _lock_setup_entered = False + commit_fence.finish_lock_setup() + if _lock_db is not None and _lock_sid: _lock_holder = _compression_lock_holder(agent) if _lock_lookup_error is not None: @@ -1381,6 +2424,27 @@ def _complete_compaction_lifecycle() -> None: ) _lock_acquired = True # acquired-but-unlocked compatibility path else: + if commit_fence is not None: + _lock_setup_entered = commit_fence.begin_lock_setup() + if not _lock_setup_entered: + logger.info( + "Compression commit cancelled before lock acquisition " + "(session=%s).", + agent.session_id or "none", + ) + agent._last_compaction_in_place = False + _existing_sp = getattr(agent, "_cached_system_prompt", None) + if not _existing_sp: + _existing_sp = agent._build_system_prompt(system_message) + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class="commit_fence_cancelled", + ) + _complete_compaction_lifecycle() + return messages, _existing_sp try: _lock_acquired = _try_acquire_lock( _lock_sid, _lock_holder, ttl_seconds=_lock_ttl @@ -1408,6 +2472,7 @@ def _complete_compaction_lifecycle() -> None: ) _lock_acquired = False if not _lock_acquired: + _finish_lock_setup() try: existing = _lock_db.get_compression_lock_holder(_lock_sid) except Exception: @@ -1452,24 +2517,83 @@ def _complete_compaction_lifecycle() -> None: _complete_compaction_lifecycle() return messages, _existing_sp _lock_released = False + _lock_release_guard = threading.Lock() - def _release_lock() -> None: - """Release the lock keyed on the OLD session_id (before rotation).""" + def _release_lock_holder_only() -> None: + """Stop this holder's refresher and release only its durable lock. + + Holder-qualified and idempotent (#76354 F4, from PR #71569): safe for + the HOST to invoke after a timeout without an ABA race — the DB + release is scoped to this worker's holder token, so a NEW holder's + lease can never be deleted by this stale release. + """ nonlocal _lock_released - _complete_compaction_lifecycle() - if _lock_released: - return - _lock_released = True - if _lock_refresher is not None: - try: - _lock_refresher.stop() - except Exception as _stop_err: - logger.debug("compression lock refresher stop failed: %s", _stop_err) - if _lock_db is not None and _lock_sid and _lock_holder: + with _lock_release_guard: + if _lock_released: + return + _lock_released = True + if getattr(agent, "_active_compression_lock_holder", None) == _lock_holder: + agent._active_compression_lock_holder = None + if _lock_refresher is not None: + try: + _lock_refresher.stop() + except Exception as _stop_err: + logger.debug("compression lock refresher stop failed: %s", _stop_err) + if _lock_db is not None and _lock_sid and _lock_holder: + try: + _lock_db.release_compression_lock(_lock_sid, _lock_holder) + except Exception as _rel_err: + logger.debug("compression lock release failed: %s", _rel_err) + + def _release_lock() -> None: + """Finish lifecycle cleanup and release the OLD session lock once.""" + try: + _complete_compaction_lifecycle() + finally: try: - _lock_db.release_compression_lock(_lock_sid, _lock_holder) - except Exception as _rel_err: - logger.debug("compression lock release failed: %s", _rel_err) + _release_lock_holder_only() + finally: + try: + if commit_fence is not None: + commit_fence.clear_cancelled_lock_release( + _release_lock_holder_only + ) + finally: + _finish_lock_setup() + + if _lock_holder is not None: + agent._active_compression_lock_holder = _lock_holder + if ( + commit_fence is not None + and commit_fence.register_cancelled_lock_release( + _release_lock_holder_only + ) + ): + # Cancellation already won while we were inside lock setup: the + # hook just ran synchronously, our lease is gone — abort before + # any summary work. + logger.info( + "Compression commit cancelled before summary dispatch " + "(session=%s).", + agent.session_id or "none", + ) + agent._last_compaction_in_place = False + _existing_sp = getattr(agent, "_cached_system_prompt", None) + if not _existing_sp: + _existing_sp = agent._build_system_prompt(system_message) + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class="commit_fence_cancelled", + ) + _release_lock() + return messages, _existing_sp + + # Publish the holder-qualified release hook before a timeout can win the + # fence. If no durable lock was acquired there is no hook to publish. + _finish_lock_setup() # A delayed contender can acquire the parent lock after the winning path # has released it and completed rotation. The lock serializes work but does @@ -1493,24 +2617,57 @@ def _release_lock() -> None: _existing_sp = agent._build_system_prompt(system_message) return messages, _existing_sp if _parent_already_rotated: - logger.info( - "compression skipped: session=%s was already rotated by " - "another compression path", - _lock_sid, + recovered_messages = _adopt_live_compression_child( + agent, _lock_db, _lock_sid ) _release_lock() _existing_sp = getattr(agent, "_cached_system_prompt", None) if not _existing_sp: _existing_sp = agent._build_system_prompt(system_message) + if recovered_messages is not None: + logger.warning( + "compression recovery: stale session=%s adopted live child=%s", + _lock_sid, + agent.session_id, + ) + return recovered_messages, _existing_sp + logger.warning( + "compression skipped: session=%s was already rotated by " + "another compression path, but no unique live child could be adopted", + _lock_sid, + ) return messages, _existing_sp + # Snapshot the authoritative durable cooldown only after this attempt owns + # the session lease. This runs for force=True too, but does not apply the + # automatic breaker gate: manual compression still retries immediately. + _durable_cooldown_authoritative, _durable_cooldown_state = ( + _capture_authoritative_cooldown_under_lease( + agent.context_compressor, + _compressor_attempt_snapshot, + ) + ) + if _durable_cooldown_authoritative is False: + # A bound built-in compressor reached its durable getter and the read + # failed. Proceeding with force=True could clear an unknown newer row + # before cancellation has enough information to restore it. This is a + # persistence-safety abort, not automatic breaker gating. + _release_lock() + existing_prompt = getattr(agent, "_cached_system_prompt", None) + if not existing_prompt: + existing_prompt = agent._build_system_prompt(system_message) + return messages, existing_prompt + # The agent may have been constructed before another path completed an # in-place compaction on the same session. Re-read durable breaker state # after acquiring the session lock so this final gate cannot act on the # stale snapshot loaded by bind_session_state(). if not force: compressor = agent.context_compressor - _refresh_persisted_compression_guards(compressor) + _refresh_persisted_compression_guards( + compressor, + include_cooldown=False, + ) blocked = getattr( type(compressor), "_automatic_compression_blocked", @@ -1524,16 +2681,65 @@ def _release_lock() -> None: return messages, existing_prompt _activity_heartbeat: Optional[_CompressionActivityHeartbeat] = None + messages_before_compression = None try: if _lock_holder is not None: - _lock_refresher = _CompressionLockLeaseRefresher( + _candidate_refresher = _CompressionLockLeaseRefresher( _lock_db, _lock_sid, _lock_holder, _lock_ttl, _lock_refresh_interval, ) - _lock_refresher.start() + # Cancellation may release the holder after hook publication but + # before this refresher starts. Serialize that check/start with + # the idempotent release path so a refresher is never started for + # an already-released lock (#76354 F4 / PR #71569). + with _lock_release_guard: + if not _lock_released: + _lock_refresher = _candidate_refresher + _lock_refresher.start() + + # The caller's history snapshot predates lease acquisition. Reload the + # durable parent after the lease is live; MORE durable rows than the + # snapshot carries means a frontend/background writer committed a turn + # in that window, so publishing from this snapshot would omit it. + # Deliberately a LENGTH check, not content equality: in-memory + # mutation of past turns is legal (multimodal compression, retry + # history replacement, think-tag stripping), and a content-equality + # abort would permanently wedge compression on such sessions — the + # #14694 failure shape. + # Rotation-only: in-place compaction (archive_and_compact) is + # non-destructive — pre-compaction rows are soft-archived (active=0, + # compacted=1), stay searchable and recoverable, so snapshot/durable + # drift cannot lose data there and must not abort compaction. + # + # When durable DID grow, ADOPT it and continue rather than aborting. + # Aborting returned the stale snapshot unchanged, so busy sessions + # (memory review / shared session_id writers) stayed permanently + # behind the DB: every /compress and auto-compress saw + # "changed before lease acquisition", surfaced as the misleading + # "No changes from compression", and never reclaimed tokens. + if not in_place and _lock_db is not None and _lock_sid: + durable_loader = getattr( + type(_lock_db), "get_messages_as_conversation", None + ) + if callable(durable_loader): + durable_parent = durable_loader(_lock_db, _lock_sid) + if isinstance(durable_parent, list) and len(durable_parent) > len(messages): + logger.info( + "compression: session=%s grew before lease " + "(%d → %d msgs); adopting durable snapshot", + _lock_sid, + len(messages), + len(durable_parent), + ) + messages = durable_parent + _pre_msg_count = len(messages) + # Token estimate was for the stale snapshot; clear it so + # the compressor re-derives from the adopted transcript + # instead of under-counting the newly visible rows. + approx_tokens = 0 # Notify external memory provider before compression discards context. # The provider's on_pre_compress() may return a string of insights it @@ -1574,8 +2780,129 @@ def _release_lock() -> None: ) messages_before_compression = copy.deepcopy(messages) - _activity_heartbeat = _CompressionActivityHeartbeat(agent).start() - compressed = compress_fn(messages, **compress_kwargs) + _activity_heartbeat = _CompressionActivityHeartbeat( + agent, commit_fence=commit_fence + ).start() + # Publish forward progress to the commit fence while the summary LLM + # call streams. Async hosts (gateway session hygiene) poll + # ``commit_fence.seconds_since_progress()`` to extend their deadline + # while tokens are moving — so a SLOW summary model is only killed + # when it is actually silent, not merely thorough. The hook is + # thread-local and the compress call is synchronous on this thread, + # so it cannot leak into unrelated auxiliary calls. + # + # Callers that pass no commit_fence install a no-op progress hook + # here. AIAgent._compress_context injects an owned fence for + # fenceless callers so the host-level progress-aware wait can + # extend on streamed tokens; gateway hygiene already passes its + # own fence. An ACTIVE hook (even a no-op) is what switches the + # summary call onto the streamed path — giving every compression + # path the same two guarantees: the configured timeout acts on + # inactivity (slow models finish), and a byte-trickling provider + # that keeps the connection alive forever is cut off at the + # streamed total ceiling (see _aux_stream_total_ceiling) instead of + # outliving the SDK's inactivity timeout indefinitely. + from agent.auxiliary_client import ( + aux_interrupt_protection, + aux_progress_hook, + ) + _progress_hook = ( + commit_fence.touch_progress if commit_fence is not None + else (lambda: None) + ) + # F4 state-ordering (#76354): a LATE successful summary must not undo + # the timeout cooldown the host recorded. Install a cancellation + # check the compressor consults BEFORE clearing the failure cooldown; + # removed in the finally below so it cannot leak into later attempts + # (e.g. a manual /compress force-clear). + if commit_fence is not None: + try: + agent.context_compressor._compression_cancelled_check = ( + lambda: commit_fence.is_cancelled + ) + except Exception: + pass + # Incoming-message interrupts and active-turn redirects must not tear an + # atomic summary in half (#23975). Explicit stop surfaces set a separate + # Event atomically; never infer cause from the racy message fields. + _hard_cancel_event = getattr(agent, "_hard_interrupt_requested", None) + try: + # F6: never start expensive summary work for an already-cancelled + # fence (a stale queued job admitted after host departure). + if commit_fence is not None and commit_fence.is_cancelled: + logger.info( + "Compression cancelled before summary dispatch " + "(session=%s) — skipping summary work.", + agent.session_id or "none", + ) + compressed = messages + else: + with aux_progress_hook(_progress_hook), aux_interrupt_protection( + cancel_event=_hard_cancel_event + ): + compressed = compress_fn(messages, **compress_kwargs) + # Freeze a hard stop that arrived after the final provider + # attempt unwound but before this transaction can rotate + # session state. + if ( + _hard_cancel_event is not None + and _hard_cancel_event.is_set() + ): + raise AuxiliaryExplicitCancellation() + finally: + if commit_fence is not None: + try: + agent.context_compressor._compression_cancelled_check = None + except Exception: + pass + except AuxiliaryExplicitCancellation: + try: + _restore_compressor_attempt_state( + agent.context_compressor, + _compressor_attempt_snapshot, + durable_cooldown_authoritative=_durable_cooldown_authoritative, + durable_cooldown_state=_durable_cooldown_state, + ) + except BaseException as _rollback_exc: + # Compensation failure must surface, but it must not strand the + # session lease or retain an in-memory transcript mutation. + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) + if _activity_heartbeat is not None: + _activity_heartbeat.stop("context compression rollback failed") + _activity_heartbeat = None + _release_lock() + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class=f"rollback:{type(_rollback_exc).__name__}", + ) + raise + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) + if _activity_heartbeat is not None: + _activity_heartbeat.stop("context compression cancelled") + _activity_heartbeat = None + _release_lock() + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class="explicit_interrupt", + ) + _existing_sp = getattr(agent, "_cached_system_prompt", None) + if not _existing_sp: + _existing_sp = agent._build_system_prompt(system_message) + return messages, _existing_sp except BaseException as _compress_exc: # ANY exception after lock acquisition — memory hook, capability # inspection, engine lookup, or compress() — must release the lock so @@ -1608,6 +2935,9 @@ def _release_lock() -> None: _compression_used_fallback = bool( getattr(agent.context_compressor, "_last_summary_fallback_used", False) ) + _compression_feasibility_skip = bool( + getattr(agent.context_compressor, "_last_feasibility_skip", False) + ) # If compression aborted (aux LLM failed to produce a usable summary) # the compressor returns the input messages unchanged. Surface the @@ -1685,8 +3015,19 @@ def _release_lock() -> None: return messages, _existing_sp if commit_fence is not None: - _commit_fence_entered = commit_fence.begin_commit() + _commit_fence_entered = commit_fence.begin_commit(_hard_cancel_event) if not _commit_fence_entered: + _restore_compressor_attempt_state( + agent.context_compressor, + _compressor_attempt_snapshot, + durable_cooldown_authoritative=_durable_cooldown_authoritative, + durable_cooldown_state=_durable_cooldown_state, + ) + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) logger.info( "Compression commit cancelled before session mutation " "(session=%s).", @@ -1803,6 +3144,20 @@ def _release_lock() -> None: ): new_system_prompt = cached_system_prompt agent._cached_system_prompt = cached_system_prompt + # _invalidate_system_prompt() above also cleared the + # cross-session-stable prefix marker boundary. The kept prompt + # is byte-identical, so reconstruct the stable tier and reuse + # it ONLY when the kept prompt still literally starts with it + # (same startswith gate as the restore path); otherwise the + # request layer falls back to the legacy single-breakpoint + # layout with the prompt bytes untouched. + from agent.system_prompt import reconstruct_static_prefix + + reconstruct_static_prefix( + agent, + system_message=system_message, + log_label="compression keep-prompt", + ) else: new_system_prompt = agent._build_system_prompt(system_message) agent._cached_system_prompt = new_system_prompt @@ -1879,81 +3234,54 @@ def _release_lock() -> None: ) except Exception: pass # best-effort — don't block compression on a flush error - # Propagate title to the new session with auto-numbering + # Publish parent closure + child row + compacted handoff in + # one transaction. No reader can observe a missing/empty child. + # The rotation child must stay on the parent's profile — + # mirror _ensure_db_session's stamp ("default" persists as + # NULL). publish_compression_child additionally COALESCEs + # from the parent row, covering app-global remote sessions + # whose thread lacks the HERMES_HOME context. + try: + from hermes_cli.profiles import get_active_profile_name + + _profile_for_child = get_active_profile_name() + if _profile_for_child == "default": + _profile_for_child = None + except Exception: + _profile_for_child = None old_title = agent._session_db.get_session_title(agent.session_id) - agent._session_db.end_session(agent.session_id, "compression") old_session_id = agent.session_id - agent.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}" - # Ordering contract: the agent thread updates the contextvar here; - # the gateway propagates to SessionEntry after run_in_executor returns. + new_session_id = ( + f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_" + f"{uuid.uuid4().hex[:6]}" + ) + agent._session_db.publish_compression_child( + parent_session_id=old_session_id, + child_session_id=new_session_id, + source=agent.platform + or os.environ.get("HERMES_SESSION_SOURCE", "cli"), + model=agent.model, + model_config=agent._session_init_model_config, + system_prompt=new_system_prompt, + messages=compressed, + cwd=getattr(agent, "working_directory", None), + profile_name=_profile_for_child, + compression_lock_holder=_lock_holder, + require_compression_lease=_lock_holder is not None, + ) + agent.session_id = new_session_id try: from gateway.session_context import set_current_session_id set_current_session_id(agent.session_id) except Exception: os.environ["HERMES_SESSION_ID"] = agent.session_id - # The gateway/tools session context (ContextVar + env) and the - # logging session context are SEPARATE mechanisms. The call above - # moves the former; the ``[session_id]`` tag on log lines comes - # from ``hermes_logging._session_context`` (set once per turn in - # conversation_loop.py). Without this, post-rotation log lines in - # the same turn keep the STALE old id while the message/DB/gateway - # state carry the new one — breaking log correlation exactly at the - # compaction boundary (see #34089). Guarded separately so a logging - # failure can never regress the routing update above. try: from hermes_logging import set_session_context set_session_context(agent.session_id) except Exception: pass - agent._session_db_created = False - try: - agent._session_db.create_session( - session_id=agent.session_id, - source=agent.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"), - model=agent.model, - model_config=agent._session_init_model_config, - parent_session_id=old_session_id, - ) - except Exception as _cs_err: - # The child row could not be created (e.g. FK constraint, - # contended write). Previously the outer handler simply - # warned and let the agent continue on the NEW id — which - # has no row in state.db, producing an orphan: the parent - # is ended, the child is never indexed, and every - # subsequent message is attributed to a session that - # doesn't exist (#33906/#33907). Roll the live id back to - # the parent so the conversation stays attached to a real, - # indexed session instead of a phantom. - logger.warning( - "Compression child session create failed (%s) — " - "rolling back to parent session %s to avoid an orphan.", - _cs_err, old_session_id, - ) - agent.session_id = old_session_id - try: - from gateway.session_context import set_current_session_id - set_current_session_id(agent.session_id) - except Exception: - os.environ["HERMES_SESSION_ID"] = agent.session_id - try: - from hermes_logging import set_session_context - set_session_context(agent.session_id) - except Exception: - pass - # Re-open the parent: it was ended above, but we're - # continuing on it, so it must not stay closed. - try: - agent._session_db.reopen_session(old_session_id) - except Exception: - pass - old_session_id = None # no rotation happened - # The parent row already exists in state.db, so mark the - # session as created — _ensure_db_session would otherwise - # retry a (harmless INSERT OR IGNORE) create next turn. - agent._session_db_created = True - raise agent._session_db_created = True split_status = "rotated_committed" # Carry a persistent /goal onto the continuation session. @@ -1973,18 +3301,14 @@ def _release_lock() -> None: except (ValueError, Exception) as e: logger.debug("Could not propagate title on compression: %s", e) - # Shared post-write steps (both modes target agent.session_id, which - # in-place keeps and rotation has already reassigned to the new id): - # refresh the stored system prompt and reset the flush cursor so the - # next turn re-bases its append diff. - agent._session_db.update_system_prompt(agent.session_id, new_system_prompt) + # In-place mode still updates/replaces the current row here. + # Rotation already published prompt + compacted handoff atomically. if in_place: + agent._session_db.update_system_prompt( + agent.session_id, new_system_prompt + ) agent._last_flushed_db_idx = 0 else: - # A headless turn can be killed before its finalizer. Persist - # the rotated child's compacted handoff at the boundary so - # the new session is immediately resumable. - agent._session_db.replace_messages(agent.session_id, compressed) agent._last_flushed_db_idx = len(compressed) agent._flushed_db_message_session_id = agent.session_id agent._flushed_db_message_ids = { @@ -1994,7 +3318,22 @@ def _release_lock() -> None: } _session_commit_succeeded = True except Exception as e: - split_status = "aborted" if locals().get("old_session_id") is None and not in_place else "failed_not_indexed" + if ( + not in_place + and locals().get("old_session_id") + and agent.session_id == old_session_id + ): + # Atomic publication failed (including lease loss): keep the + # parent live and discard the stale compacted snapshot. + old_session_id = None + messages[:] = copy.deepcopy(messages_before_compression) + compressed = messages + _compression_made_progress = False + split_status = ( + "aborted" + if locals().get("old_session_id") is None and not in_place + else "failed_not_indexed" + ) # If the rotation rolled back to the parent (orphan-avoidance # above), agent.session_id is the still-indexed parent and # old_session_id was cleared — so this is recovery, not an @@ -2019,6 +3358,31 @@ def _release_lock() -> None: ) _boundary_parent = _old_sid or agent.session_id or "" + # Round-2 #4: the activity heartbeat's terminal "context compression + # completed" stamp landed on the PARENT row (force-persisted before + # the rotation re-pointed agent.session_id at the child). Without a + # cleanup, the archived parent advertises a fresh last_activity_at + + # "context compression completed" forever — a permanent false-fresh + # row for any activity consumer that scans ended sessions. Clear the + # labels on the parent best-effort (keeps last_activity_at so idle + # clocks stay continuous; the CHILD carries the live labels). + if _old_sid and _session_commit_succeeded: + try: + _labels_db = getattr(agent, "_session_db", None) + _clear_labels = getattr( + type(_labels_db) if _labels_db is not None else None, + "clear_session_activity_labels", + None, + ) + if callable(_clear_labels): + _clear_labels(_labels_db, _old_sid) + except Exception: + logger.debug( + "failed to clear archived compression parent's activity " + "labels (ignored)", + exc_info=True, + ) + # Notify the context engine that a compaction boundary occurred. Plugin # engines (e.g. hermes-lcm) use boundary_reason="compression" to preserve # DAG lineage / checkpoint per-session state across the boundary instead of @@ -2120,6 +3484,7 @@ def _release_lock() -> None: record_boundary( agent.context_compressor, used_fallback=_compression_used_fallback, + feasibility_skip=_compression_feasibility_skip, ) else: agent.context_compressor._verify_compaction_cleared_threshold = True @@ -2132,6 +3497,13 @@ def _release_lock() -> None: reset_file_dedup(task_id) except Exception: pass + # Same for the skill_view repeat-view dedup: a post-compression + # re-view must return the full skill content again. + try: + from tools.skills_tool import reset_skill_view_dedup + reset_skill_view_dedup(task_id) + except Exception: + pass logger.info( "context compression done: session=%s messages=%d->%d rough_tokens=~%s awaiting_real_usage=true", @@ -2493,16 +3865,28 @@ def _source_to_data_url(source: Any) -> Optional[str]: media_type = "image/jpeg" return f"data:{media_type};base64,{data}" - def _write_data_url_to_source(source: dict, data_url: str) -> None: + def _write_data_url_to_source(source: dict, data_url: str) -> dict: + """Return a NEW source dict carrying the re-encoded payload. + + Copy-on-write: content parts on the per-call ``api_messages`` list may + be shared references into the persistent conversation history (the + per-message copy is shallow, and cache decoration only deep-copies the + marked messages). Mutating the existing dict would rewrite the stored + transcript with the degraded image — so the caller replaces the part, + never edits it in place. + """ header, _, data = data_url.partition(",") media_type = "image/jpeg" if header.startswith("data:"): candidate = header[len("data:"):].split(";", 1)[0].strip() if candidate.startswith("image/"): media_type = candidate - source["type"] = "base64" - source["media_type"] = media_type - source["data"] = data + return { + **source, + "type": "base64", + "media_type": media_type, + "data": data, + } for msg in api_messages: if not isinstance(msg, dict): @@ -2510,7 +3894,13 @@ def _write_data_url_to_source(source: dict, data_url: str) -> None: content = msg.get("content") if not isinstance(content, list): continue - for part in content: + # Copy-on-write per message: never mutate part/source dicts in place — + # they can alias the stored conversation history (see + # _write_data_url_to_source). Build a replacement content list on the + # first shrunken part and reassign msg["content"] (a top-level write on + # the per-call message copy, which never reaches history). + new_content: list | None = None + for part_idx, part in enumerate(content): if not isinstance(part, dict): continue ptype = part.get("type") @@ -2519,7 +3909,12 @@ def _write_data_url_to_source(source: dict, data_url: str) -> None: url = _source_to_data_url(source) resized, unshrinkable = _shrink_data_url(url or "") if resized and isinstance(source, dict): - _write_data_url_to_source(source, resized) + if new_content is None: + new_content = list(content) + new_content[part_idx] = { + **part, + "source": _write_data_url_to_source(source, resized), + } changed_count += 1 elif unshrinkable: unshrinkable_oversized += 1 @@ -2533,17 +3928,26 @@ def _write_data_url_to_source(source: dict, data_url: str) -> None: url = image_value.get("url", "") resized, unshrinkable = _shrink_data_url(url) if resized: - image_value["url"] = resized + if new_content is None: + new_content = list(content) + new_content[part_idx] = { + **part, + "image_url": {**image_value, "url": resized}, + } changed_count += 1 elif unshrinkable: unshrinkable_oversized += 1 elif isinstance(image_value, str): resized, unshrinkable = _shrink_data_url(image_value) if resized: - part["image_url"] = resized + if new_content is None: + new_content = list(content) + new_content[part_idx] = {**part, "image_url": resized} changed_count += 1 elif unshrinkable: unshrinkable_oversized += 1 + if new_content is not None: + msg["content"] = new_content if changed_count: logger.info( diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index e737344f62f9..dd8529800aef 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -22,9 +22,7 @@ import random import re import ssl -import threading import time -import uuid from typing import Any, Dict, List, Optional from agent.codex_responses_adapter import _summarize_user_message_for_log @@ -40,7 +38,6 @@ from agent.context_engine import automatic_compaction_status_message from agent.display import KawaiiSpinner from agent.error_classifier import FailoverReason, classify_api_error -from agent.iteration_budget import IterationBudget from agent.turn_context import ( _compression_warrants_another_preflight_pass, build_turn_context, @@ -48,6 +45,7 @@ reanchor_current_turn_user_idx, ) from agent.turn_retry_state import TurnRetryState +from agent.runtime_cwd import resolve_agent_cwd from agent.message_sanitization import ( close_interrupted_tool_sequence, _repair_tool_call_arguments, @@ -71,7 +69,11 @@ save_context_length, ) from agent.process_bootstrap import _install_safe_stdio -from agent.prompt_caching import apply_anthropic_cache_control +from agent.prompt_caching import ( + build_prompt_cache_plan, + strip_anthropic_cache_control, + strip_anthropic_tool_cache_control, +) from agent.retry_utils import ( adaptive_rate_limit_backoff, is_zai_coding_overload_error, @@ -117,22 +119,44 @@ def _apply_active_turn_redirect(agent: Any, messages: List[Dict[str, Any]], text Incomplete provider reasoning blocks are not valid replay items (Anthropic signs them; Responses reasoning items require their following output). - Preserve only what Hermes actually displayed, demoted to ordinary text, - then add the correction as a real user message. This keeps role alternation + Preserve only the *visible* response text, demoted to ordinary text, then + add the correction as a real user message. This keeps role alternation valid and leaves every previously cached message byte-for-byte unchanged. + + INVARIANT — raw chain-of-thought must never be serialized into replayable + message content. Streamed reasoning is display-only state: it may be shown + live, but it does not re-enter the transcript as assistant (or user) text. + An assistant turn whose content inlines its own chain-of-thought reads to + Anthropic's output classifier as reasoning-injection/prefill jailbreak, + and because the poisoned checkpoint is persisted and replayed on every + subsequent call, the session dies permanently with deterministic + "Provider returned an empty response" storms that no retry, nudge, or + empty-recovery branch can escape (July 2026: four sessions bricked this + way; every reasoning-free checkpoint that week was untouched — same + mechanism as the ~/.hermes/prefill.json incident, 20/20 blocked with + assistant-exposed CoT vs 0/20 without). The interrupted reasoning was + incomplete by definition; the model regenerates it on the retried turn. + If a future path needs to preserve interrupted thinking, carry it in a + provider-gated reasoning *field*, never in content. + INVARIANT — the scaffolding is provider-replay text, not transcript text. + ``[This response was interrupted by a user correction.]`` and its + ``Visible response before the interruption:`` header exist so the MODEL + understands its own reply was cut off. They are not prose the user wrote + or the agent said. Persisting them into ``content`` painted the raw + machinery as an assistant bubble on every reload (and merged it into the + preceding tool-call bubble), which is what made a steered transcript + unreadable. Carry the scaffolded form in the ``api_content`` sidecar -- + the exact bytes replayed to the provider -- and keep ``content`` clean. + When nothing was on screen there is no clean form at all, so the row is + marked ``display_kind="hidden"``: still replayed to the model, dropped by + every transcript surface (desktop, TUI, CLI resume), exactly like the + compaction-reference rows. """ - reasoning = str( - getattr(agent, "_current_streamed_reasoning_text", "") or "" - ).strip() visible = agent._strip_think_blocks( getattr(agent, "_current_streamed_assistant_text", "") or "" ).strip() checkpoint_parts = ["[This response was interrupted by a user correction.]"] - if reasoning: - checkpoint_parts.extend( - ["Reasoning shown before the interruption:", reasoning] - ) if visible: checkpoint_parts.extend( ["Visible response before the interruption:", visible] @@ -149,16 +173,76 @@ def _apply_active_turn_redirect(agent: Any, messages: List[Dict[str, Any]], text f"{checkpoint}\n\n" f"{text}" ) - messages.append({"role": "user", "content": correction}) + # Transcript shows the user's own words; the provider replays the + # scaffolded form so it still sees the interrupted context. + messages.append( + {"role": "user", "content": text, "api_content": correction} + ) else: - messages.append({"role": "assistant", "content": checkpoint}) + entry: Dict[str, Any] = { + "role": "assistant", + "content": visible or checkpoint, + "api_content": checkpoint, + } + if not visible: + # Nothing reached the screen — this row carries no assistant prose + # at all, only the cut-off notice for the model. + entry["display_kind"] = "hidden" + messages.append(entry) messages.append({"role": "user", "content": text}) agent._current_streamed_assistant_text = "" - agent._current_streamed_reasoning_text = "" agent._stream_needs_break = True +def _is_copilot_provider(agent: Any) -> bool: + """Delegate to ``AIAgent._is_copilot_provider`` (single owner of the check). + + ``agent.provider`` is not always the normalized ``copilot`` slug — + ``/model`` and profile configs can leave the alias ``github-copilot`` (or + ``github``) in place, and a bare ``provider == "copilot"`` gate silently + skips credential recovery for those spellings. + """ + try: + return bool(agent._is_copilot_provider()) + except Exception: + return (getattr(agent, "provider", "") or "").strip().lower() in { + "copilot", + "github-copilot", + "github", + } + + +def _is_stale_copilot_credential_error(status_code: Optional[int], error_message: str) -> bool: + """Detect a Copilot 400 that is really a STALE / DEGRADED credential. + + Copilot surfaces a stale or degraded credential as an HTTP 400 rather than a + clean 401. Two body markers indicate this class: + + - ``model_not_available_for_integrator`` — the request reached the + restricted ``copilot-language-server`` integrator (the server's fallback + when it receives a raw OAuth token instead of an exchanged API token), + whose model allowlist omits enterprise-only models. + - ``model_not_supported`` / "the requested model is not supported" — the + cached bearer's Copilot entitlement rotated out from under a long-lived + process. + + Matched narrowly (status 400 AND a specific marker) so a genuinely wrong + model name — a real 400 — never triggers the single-shot re-exchange. The + caller enforces copilot-provider scoping and the single-shot guard. + """ + lowered = (error_message or "").lower() + is_400 = status_code == 400 or "error code: 400" in lowered + if not is_400: + return False + return ( + "model_not_available_for_integrator" in lowered + or "not available for integrator" in lowered + or "model_not_supported" in lowered + or "the requested model is not supported" in lowered + ) + + def _image_error_max_dimension(error: Exception) -> Optional[int]: """Extract a provider-reported image dimension ceiling, if present.""" parts = [] @@ -436,6 +520,20 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # Continuing session — reuse the exact system prompt from the # previous turn so the Anthropic cache prefix matches. agent._cached_system_prompt = stored_prompt + # Reconstruct the cross-session-stable prefix for the early cache + # breakpoint. The static prefix is not persisted (only the full + # prompt is), so gateway surfaces that build a fresh AIAgent per + # turn would otherwise lose the two-block system layout after the + # first turn — flip-flopping the wire shape mid-conversation and + # silently degrading to the legacy single-breakpoint layout. + # + # ``reconstruct_static_prefix`` gates on ``_use_prompt_caching`` (so + # non-Anthropic routes skip the rebuild), applies the startswith + # safety gate (stored prompt bytes are never rewritten), and + # fails open to the legacy cache layout. + from agent.system_prompt import reconstruct_static_prefix + + reconstruct_static_prefix(agent, system_message=system_message) return if stored_prompt: stored_state = "stale_runtime" @@ -468,7 +566,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # session is created (not on continuation). Plugins can use this # to initialise session-scoped state (e.g. warm a memory cache). try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_start", session_id=agent.session_id, @@ -508,9 +606,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) def _stored_prompt_matches_runtime(agent, prompt: str) -> bool: - """Return False when the persisted Model/Provider lines are stale.""" + """Return False when the persisted runtime-identity lines are stale.""" def line_value(label: str) -> str: + """Last matching line wins. + + Safe ONLY for fields emitted in the volatile tier at the very END of + the prompt (Model / Provider / Platform). User-supplied project + context (AGENTS.md / CLAUDE.md / .cursorrules) is embedded in the + middle context tier, so a last-match scan lets project prose shadow + any field emitted EARLIER — see ``host_info_value``. + """ prefix = f"{label}:" value = "" for line in prompt.splitlines(): @@ -518,6 +624,32 @@ def line_value(label: str) -> str: value = line[len(prefix):].strip() return value + def host_info_value(label: str) -> str: + """Read a field from the prompt's own host-info block. + + The host-info block (``build_environment_hints``) sits in the STABLE + tier, ahead of the embedded project context files. A bare scan of the + whole prompt would therefore match a user's ``AGENTS.md`` that merely + contains a line starting with the same label, comparing runtime state + against project prose. That mismatch never clears, so the check would + reject the stored prompt on EVERY turn — rebuilding the system prompt + each message and destroying the prefix cache for the whole session, + which is far worse than the staleness this function guards against. + + Anchor on the ``User home directory:`` line that immediately precedes + the working-directory line in that block, and take the FIRST such + occurrence, so only Hermes' own emitted block can satisfy the read. + """ + prefix = f"{label}:" + lines = prompt.splitlines() + for idx, line in enumerate(lines): + if not line.startswith("User home directory:"): + continue + for candidate in lines[idx + 1: idx + 4]: + if candidate.startswith(prefix): + return candidate[len(prefix):].strip() + return "" + stored_model = line_value("Model") current_model = str(getattr(agent, "model", "") or "").strip() if stored_model and current_model and stored_model != current_model: @@ -528,6 +660,24 @@ def line_value(label: str) -> str: if stored_provider and current_provider and stored_provider != current_provider: return False + # Detect cwd drift: if the stored prompt was built in a different working + # directory, reuse would silently inject a stale path into the prefix cache. + # Compare against resolve_agent_cwd() — the SAME resolver used to build the + # prompt — so gateway/TUI sessions that set TERMINAL_CWD are not falsely + # rejected (they would always differ from the launch dir's os.getcwd()). + stored_cwd = host_info_value("Current working directory") + if stored_cwd: + if stored_cwd != str(resolve_agent_cwd()): + return False + + # Detect runtime-surface drift: the stored prompt records which platform it + # was built for (e.g. "desktop" vs "cli"). Reusing a desktop-built prompt on + # a terminal session (or vice versa) would inject the wrong runtime hints. + stored_platform = line_value("Platform") + current_platform = str(getattr(agent, "platform", "") or "").strip() + if stored_platform and current_platform and stored_platform != current_platform: + return False + return True @@ -588,6 +738,92 @@ def _get_continuation_prompt(is_partial_stub: bool, dropped_tools: Optional[List ) +# Memo for the send-path tool-call argument canonicalization inside +# run_conversation(). That pass re-canonicalizes the arguments string of +# EVERY historical tool call on EVERY API-call iteration (quadratic in +# session tool-call count), and the api_messages copies share the exact +# argument string objects with the persisted history, so the same strings +# come through unchanged iteration after iteration. +# +# Soundness: canonicalization is a pure, deterministic function of the +# input string (fixed separators, sort_keys=True), so a value-keyed memo +# is exact — equal inputs always produce the canonical form computed the +# first time. Malformed strings raise out of json.loads BEFORE anything +# is stored, so the repair fallback below is never memoized and reruns on +# every occurrence, exactly as before. Bounded FIFO eviction mirrors the +# _MSG_TOKENS_CACHE idiom in agent/model_metadata.py. +_CANON_ARGS_CACHE: Dict[str, str] = {} +_CANON_ARGS_CACHE_MAX = 4096 +# Count bound alone doesn't bound MEMORY: write_file/patch argument strings +# run 100KB+, so 4096 entries could pin ~800MB in a long-lived gateway +# process. The byte budget keeps the memo effective for the common case +# (args ~0.5-2KB) while bounding the worst case. +_CANON_ARGS_CACHE_MAX_BYTES = 32 * 1024 * 1024 +_canon_args_cache_bytes = 0 + + +def _canonicalize_tool_call_arguments(arg_str: str) -> str: + """Return the canonical wire form of a tool-call arguments JSON string. + + Raises whatever ``json.loads`` raises on malformed input; the caller + falls back to ``_repair_tool_call_arguments``, exactly as before. + """ + global _canon_args_cache_bytes + cached = _CANON_ARGS_CACHE.get(arg_str) + if cached is not None: + return cached + canonical = json.dumps( + json.loads(arg_str), separators=(",", ":"), sort_keys=True, + ) + _CANON_ARGS_CACHE[arg_str] = canonical + _canon_args_cache_bytes += len(arg_str) + len(canonical) + while len(_CANON_ARGS_CACHE) > _CANON_ARGS_CACHE_MAX or ( + _canon_args_cache_bytes > _CANON_ARGS_CACHE_MAX_BYTES + and len(_CANON_ARGS_CACHE) > 1 + ): + try: + evicted_key = next(iter(_CANON_ARGS_CACHE)) + evicted_val = _CANON_ARGS_CACHE.pop(evicted_key) + _canon_args_cache_bytes -= len(evicted_key) + len(evicted_val) + except (StopIteration, KeyError, RuntimeError): + break + return canonical + + +def _canonicalize_api_tool_calls(api_messages) -> None: + """Canonicalize tool-call argument JSON on the send-path message copy. + + Rewrites each message's ``tool_calls`` in place (copy-on-write for the + tool-call dicts it canonicalizes; the persisted history is untouched). + The pass still traverses every message and tool call each iteration; + the memo above bounds the JSON parse/serialize work to one round-trip + per UNIQUE argument string instead of one per string per iteration — + the quadratic part of the cost. The remaining traversal is pointer + chasing and dict copies, cheap next to a json.loads + json.dumps. + """ + for am in api_messages: + tcs = am.get("tool_calls") + if not tcs: + continue + new_tcs = [] + for tc in tcs: + if isinstance(tc, dict) and "function" in tc: + try: + tc = {**tc, "function": { + **tc["function"], + "arguments": _canonicalize_tool_call_arguments( + tc["function"]["arguments"] + ), + }} + except Exception: + tc["function"]["arguments"] = _repair_tool_call_arguments( + tc["function"]["arguments"], + tc["function"].get("name", "?"), + ) + new_tcs.append(tc) + am["tool_calls"] = new_tcs + + def _invalid_tool_name_error_content(name: str, valid_tool_names) -> str: """Error-result content for a tool call whose name isn't a real tool. @@ -690,6 +926,41 @@ def _compression_deferred_result( } +def _rewrite_system_content_blocks(system_message: dict, effective: str) -> bool: + """Rewrite a cache-decorated system message in place, keeping its blocks. + + ``apply_anthropic_cache_control`` runs once per call block, *before* the + retry loop, and splits the system prompt into ``[static prefix, volatile + tail]`` text blocks carrying the cache_control breakpoints. Assigning a bare + string over that list drops both breakpoints, so the failover retry ships + the whole system prompt uncached and re-bills it in full. + + ``rewrite_prompt_model_identity`` only touches the LAST ``Model:`` / + ``Provider:`` lines, and those live in the volatile tail — so the static + prefix stays byte-identical and its cache entry keeps matching. Returns + False when the shape is not one we can safely patch, so the caller falls + back to the plain-string assignment. + """ + content = system_message.get("content") + if not isinstance(content, list) or not content: + return False + if not all( + isinstance(part, dict) and part.get("type") == "text" for part in content + ): + return False + if len(content) == 1: + content[0]["text"] = effective + return True + if len(content) == 2: + head = content[0].get("text") or "" + if head and effective.startswith(head): + tail = effective[len(head):] + if tail: + content[1]["text"] = tail + return True + return False + + def _sync_failover_system_message(agent, api_messages, active_system_prompt): """Refresh the in-flight system message after a provider failover. @@ -712,10 +983,118 @@ def _sync_failover_system_message(agent, api_messages, active_system_prompt): effective = sp if agent.ephemeral_system_prompt: effective = (effective + "\n\n" + agent.ephemeral_system_prompt).strip() - api_messages[0]["content"] = effective + if not _rewrite_system_content_blocks(api_messages[0], effective): + api_messages[0]["content"] = effective return sp +def _ensure_cached_system_prompt_static(agent, system_message=None) -> None: + """Rebuild ``_cached_system_prompt_static`` when caching becomes active. + + Sessions restored under a cache-off primary skip the static-prefix rebuild + (gated on ``_use_prompt_caching`` at restore time). A later failover to a + cache-on provider would otherwise redecorate with ``static_system_prefix= + None`` and silently fall back to the legacy system-plus-3 layout (#72626). + + Thin wrapper over :func:`agent.system_prompt.reconstruct_static_prefix`, + which memoizes failed rebuilds so this stays cheap on the retry-loop hot + path (it runs at the top of every attempt). + """ + from agent.system_prompt import reconstruct_static_prefix + + reconstruct_static_prefix( + agent, system_message=system_message, log_label="failover redecoration" + ) + + +def _peel_moa_guidance( + messages: List[Dict[str, Any]], + guidance: Any, +) -> List[Dict[str, Any]]: + """Remove MoA reference guidance previously attached by ``_attach_reference_guidance``. + + Thin wrapper over :func:`agent.moa_loop.peel_reference_guidance` (kept + adjacent to the attach so the forward/inverse shapes evolve together). + Lazy import mirrors the module's other moa_loop touchpoints. + """ + from agent.moa_loop import peel_reference_guidance + + return peel_reference_guidance(messages, guidance) + + +def _redecorate_prompt_cache_for_provider( + agent, + api_messages: List[Dict[str, Any]], + *, + system_message=None, + moa_prepared: Optional[Dict[str, Any]] = None, + tools_for_api: Optional[List[Dict[str, Any]]] = None, +) -> tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]] | tuple[List[Dict[str, Any]], Optional[Dict[str, Any]], List[Dict[str, Any]]]: + """Strip and re-apply cache_control for the *current* provider policy. + + Decoration runs once per call block before the retry loop for the primary + provider. ``try_activate_fallback`` refreshes ``_use_prompt_caching`` / + ``_use_native_cache_layout`` but the nine failover ``continue`` paths reused + the old ``api_messages`` (#72626). Mirror ``_reapply_reasoning_echo_for_provider`` + by reshaping at the top of each retry attempt. + + The source list is the mutated in-flight request (image shrink / ASCII / + reasoning_details recoveries already applied), never a pristine + pre-decoration snapshot. MoA guidance is peeled and rebased without + decoration; the acting aggregator plans its resolved destination later. + """ + messages: List[Dict[str, Any]] = [ + dict(m) if isinstance(m, dict) else m for m in (api_messages or []) + ] + prepared = moa_prepared + guidance = prepared.get("guidance") if isinstance(prepared, dict) else None + if guidance: + messages = _peel_moa_guidance(messages, guidance) + + strip_anthropic_cache_control(messages) + planned_tools = strip_anthropic_tool_cache_control( + tools_for_api if tools_for_api is not None else getattr(agent, "tools", []) + ) + + if prepared is not None and getattr(agent, "provider", None) == "moa": + # Prepared MoA state is canonical: the synchronous acting-aggregator + # sender owns its destination-local cache plan after it resolves the slot. + completions = getattr(getattr(agent.client, "chat", None), "completions", None) + rebase = getattr(completions, "rebase_prepared_request", None) + if callable(rebase): + prepared = rebase(prepared, messages) + messages = prepared["messages"] + if tools_for_api is None: + return messages, prepared + return messages, prepared, planned_tools + + # Direct attribute access matches the call-block decoration site — the + # flags are unconditionally initialized on AIAgent, and a getattr + # default here would mask a real init bug as silent cache-off. + if agent._use_prompt_caching: + _ensure_cached_system_prompt_static(agent, system_message=system_message) + static = getattr(agent, "_cached_system_prompt_static", None) + direct_tool_cache = getattr( + agent, + "_direct_native_anthropic_tool_cache_capability", + lambda: False, + )() + plan = build_prompt_cache_plan( + messages, + planned_tools, + cache_ttl=agent._cache_ttl, + native_anthropic=agent._use_native_cache_layout, + static_system_prefix=static if isinstance(static, str) else None, + direct_native_tool_cache=direct_tool_cache, + ) + messages = plan.messages + planned_tools = plan.tools + + if tools_for_api is None: + return messages, prepared + return messages, prepared, planned_tools + + def _apply_context_engine_selection( agent: Any, api_messages: List[Dict[str, Any]], @@ -754,18 +1133,19 @@ def _apply_context_engine_selection( pass session_label = getattr(agent, "session_id", None) or "-" - # Pass shallow copies of every input so an engine that mutates them in - # place cannot alter the live provider request or persisted transcript - # state unless the engine explicitly returns a valid replacement. - _request_copy = [ - dict(m) if isinstance(m, dict) else m for m in api_messages - ] + # Pass shallow copies of the reference-only inputs so an engine that + # mutates them in place cannot alter persisted transcript state. Only + # ``request_messages`` (the per-call request list) is meant to be acted on, + # and it may be replaced wholesale via the return value — never mutated in + # place either. ``conversation_messages`` / ``incoming_message`` are + # read-only context; copying enforces the request-only contract rather than + # merely documenting it. _conv_copy = [dict(m) if isinstance(m, dict) else m for m in conversation_messages] \ if conversation_messages is not None else None _incoming_copy = dict(incoming_message) if isinstance(incoming_message, dict) else incoming_message try: selected = engine.select_context( - _request_copy, + api_messages, conversation_messages=_conv_copy, incoming_message=_incoming_copy, budget_tokens=getattr(engine, "context_length", 0) or 0, @@ -854,6 +1234,8 @@ def run_conversation( stream_callback: Optional[callable] = None, persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, + persist_user_display_kind: Optional[str] = None, + persist_user_display_metadata: Optional[Dict[str, Any]] = None, moa_config: Optional[dict[str, Any]] = None, ) -> Dict[str, Any]: """ @@ -872,6 +1254,13 @@ def run_conversation( synthetic prefixes. persist_user_timestamp: Optional platform event timestamp to store as metadata on that persisted user message. + persist_user_display_kind: Optional presentation type for a + synthesized user turn (``auto_continue``, ``model_switch``, …). + Display-only: transcript surfaces render the row as a timeline + event instead of a user bubble, while the model still receives + the message unchanged. + persist_user_display_metadata: Optional payload for that event + (e.g. a delegation's task count). or queuing follow-up prefetch work. Returns: @@ -914,6 +1303,8 @@ def run_conversation( stream_callback, persist_user_message, persist_user_timestamp, + persist_user_display_kind=persist_user_display_kind, + persist_user_display_metadata=persist_user_display_metadata, restore_or_build_system_prompt=_restore_or_build_system_prompt, install_safe_stdio=_install_safe_stdio, sanitize_surrogates=_sanitize_surrogates, @@ -940,6 +1331,9 @@ def run_conversation( # Commentary deduplication spans all provider continuations and tool calls # within one user turn, but must not suppress the same phrase next turn. agent._delivered_interim_texts = set() + # A configured SessionDB append failure halts only the affected turn. A + # cached gateway agent must recover on the next message if storage did. + agent._incremental_persistence_failed = False # Main conversation loop counters (pure locals consumed by the loop below). api_call_count = 0 @@ -1133,10 +1527,23 @@ def run_conversation( # However, providers like Moonshot AI require a separate 'reasoning_content' field # on assistant messages with tool_calls. We handle both cases here. request_logger = getattr(agent, "logger", None) or logging.getLogger(__name__) + # Per-agent validation cursor: skips re-json.loads-ing tool_call + # arguments on history messages already validated in a previous + # iteration. Identity-keyed (strong refs) — compression/undo/repair + # rewriting the list breaks the prefix match and forces a re-scan + # from the divergence point. See sanitize_tool_call_arguments. + _sanitize_cursor = getattr(agent, "_sanitize_args_cursor", None) + if _sanitize_cursor is None: + _sanitize_cursor = {} + try: + agent._sanitize_args_cursor = _sanitize_cursor + except Exception: + pass repaired_tool_calls = agent._sanitize_tool_call_arguments( messages, logger=request_logger, session_id=agent.session_id, + cursor=_sanitize_cursor, ) if repaired_tool_calls > 0: request_logger.info( @@ -1182,6 +1589,12 @@ def run_conversation( api_msg.pop("display_kind", None) api_msg.pop("display_metadata", None) + # Durable row identity stamped by _rows_to_conversation so the + # desktop can address a specific persisted message (reactions). + # Bookkeeping, never a provider field — only the chat-completions + # transport strips underscore keys, so drop it centrally here. + api_msg.pop("_row_id", None) + # Inject ephemeral context into the current turn's user message. # Sources: memory manager prefetch + plugin pre_llm_call hooks # with target="user_message" (the default). Both are @@ -1277,9 +1690,9 @@ def run_conversation( # # Hermes invariant: the system prompt is built ONCE per session # (cached on ``_cached_system_prompt``) and replayed verbatim on - # every turn. We send it as a single content string so the - # bytes are byte-stable across turns and upstream prompt caches - # stay warm. + # every turn. ``apply_anthropic_cache_control`` may split its stable + # prefix into content blocks on the wire, but the stored string and + # its byte-stability remain unchanged. effective_system = active_system_prompt or "" if agent.ephemeral_system_prompt: effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip() @@ -1364,19 +1777,6 @@ def run_conversation( logger=request_logger, ) - # Apply Anthropic prompt caching for Claude models on native - # Anthropic, OpenRouter, and third-party Anthropic-compatible - # gateways. Auto-detected: if ``_use_prompt_caching`` is set, - # inject cache_control breakpoints (system + last 3 messages) - # to reduce input token costs by ~75% on multi-turn - # conversations. - if agent._use_prompt_caching: - api_messages = apply_anthropic_cache_control( - api_messages, - cache_ttl=agent._cache_ttl, - native_anthropic=agent._use_native_cache_layout, - ) - # Safety net: strip orphaned tool results / add stubs for missing # results before sending to the API. Runs unconditionally — not # gated on context_compressor — so orphans from session loading or @@ -1405,29 +1805,7 @@ def run_conversation( for am in api_messages: if isinstance(am.get("content"), str): am["content"] = am["content"].strip() - for am in api_messages: - tcs = am.get("tool_calls") - if not tcs: - continue - new_tcs = [] - for tc in tcs: - if isinstance(tc, dict) and "function" in tc: - try: - args_obj = json.loads(tc["function"]["arguments"]) - tc = {**tc, "function": { - **tc["function"], - "arguments": json.dumps( - args_obj, separators=(",", ":"), - sort_keys=True, - ), - }} - except Exception: - tc["function"]["arguments"] = _repair_tool_call_arguments( - tc["function"]["arguments"], - tc["function"].get("name", "?"), - ) - new_tcs.append(tc) - am["tool_calls"] = new_tcs + _canonicalize_api_tool_calls(api_messages) # Proactively strip any surrogate characters before the API call. # Models served via Ollama (Kimi K2.5, GLM-5, Qwen) can return @@ -1435,6 +1813,49 @@ def run_conversation( # the OpenAI SDK. Sanitizing here prevents the 3-retry cycle. _sanitize_messages_surrogates(api_messages) + # NOTE (empty-content class fix): no send-time pad loop here. The + # single owner for "never send a turn strict wire validation rejects + # as empty" is ``repair_empty_non_final_messages``, which runs inside + # ``_sanitize_api_messages`` above — the unconditional pre-send + # chokepoint shared with the summary path. Its placeholder is + # non-whitespace, so it survives the whitespace-normalization pass + # regardless of ordering (a single-space pad here previously had to + # be sequenced after normalization to survive, forking the concept). + + # Build the request-local cache sections only after every transcript + # mutation. The canonical tool registry stays undecorated. + # + # Runs LAST, after every message mutation above. Marking earlier + # defeats the prefix stability the mutations exist to create: + # ``_apply_cache_marker`` rewrites ``content`` from a plain string + # into a ``[{"type": "text", ...}]`` block, so the marked messages + # no longer match the ``isinstance(content, str)`` test in the + # whitespace-normalization pass and silently keep their raw + # leading/trailing whitespace. A tool result ending in "\n" is + # therefore sent unstripped while it sits in the last-3 window and + # stripped once it rolls out of it — the same message, different + # bytes on consecutive turns, which breaks the prefix match at + # exactly the point the breakpoints were meant to protect. Marking + # last also keeps breakpoints off messages that the orphan sweep or + # the thinking-only drop is about to remove or merge away. + tools_for_api = agent.tools + if agent._use_prompt_caching and agent.provider != "moa": + _static_system_prefix = getattr(agent, "_cached_system_prompt_static", None) + _initial_cache_plan = build_prompt_cache_plan( + api_messages, + tools_for_api, + cache_ttl=agent._cache_ttl, + native_anthropic=agent._use_native_cache_layout, + static_system_prefix=( + _static_system_prefix + if isinstance(_static_system_prefix, str) + else None + ), + direct_native_tool_cache=agent._direct_native_anthropic_tool_cache_capability(), + ) + api_messages = _initial_cache_plan.messages + tools_for_api = _initial_cache_plan.tools + # Build a persistent-MoA request before measuring compression pressure. # MoA reference output is injected into the aggregator prompt, but it # is deliberately ephemeral and therefore absent from ``messages``. @@ -1765,7 +2186,26 @@ def run_conversation( # unless the active provider needs it) so the fallback request # isn't sent with stale, primary-shaped reasoning fields. agent._reapply_reasoning_echo_for_provider(api_messages) - api_kwargs = agent._build_api_kwargs(api_messages) + # Same story for prompt-cache decoration (#72626): try_activate_ + # fallback refreshes the policy flags, but the decorated list + # still carries the primary's breakpoints (or none). Strip and + # re-render for the current provider before building kwargs. + api_messages, _moa_prepared_request, tools_for_api = ( + _redecorate_prompt_cache_for_provider( + agent, + api_messages, + system_message=system_message, + moa_prepared=_moa_prepared_request, + tools_for_api=tools_for_api, + ) + ) + if tools_for_api == agent.tools: + api_kwargs = agent._build_api_kwargs(api_messages) + else: + api_kwargs = agent._build_api_kwargs( + api_messages, + tools_for_api=tools_for_api, + ) if agent._force_ascii_payload: _sanitize_structure_non_ascii(api_kwargs) if agent.api_mode == "codex_responses": @@ -1773,6 +2213,7 @@ def run_conversation( api_kwargs, allow_stream=False, is_github_responses=agent._is_copilot_url(), + sanitize_harmony_tokens=agent._is_codex_backend(), ) # Copilot x-initiator: the first API call of a user turn is # marked "user" so Copilot bills a premium request; tool-loop @@ -1806,7 +2247,7 @@ def run_conversation( _llm_middleware_trace = [] try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -1847,6 +2288,7 @@ def run_conversation( base_url=agent.base_url, api_mode=agent.api_mode, api_call_count=api_call_count, + retry_count=retry_count, request_messages=list(request_messages) if isinstance(request_messages, list) else [], @@ -1931,12 +2373,34 @@ def _perform_api_call(next_api_kwargs): next_api_kwargs, allow_stream=False, is_github_responses=agent._is_copilot_url(), + sanitize_harmony_tokens=agent._is_codex_backend(), ) if _use_streaming: return agent._interruptible_streaming_api_call( next_api_kwargs, on_first_delta=_stop_spinner ) - return agent._interruptible_api_call(next_api_kwargs) + from agent import relay_llm + + return relay_llm.execute( + next_api_kwargs, + agent._interruptible_api_call, + session_id=str(agent.session_id or ""), + name=str(agent.provider or "provider"), + model_name=str(agent.model or ""), + metadata={ + "api_mode": agent.api_mode, + "api_request_id": api_request_id, + "call_role": ( + "delegated" + if getattr(agent, "is_subagent", False) + else "fallback" + if int(getattr(agent, "_fallback_index", 0) or 0) > 0 + else "primary" + ), + "retry_count": retry_count, + }, + defer_logical_completion=True, + ) from hermes_cli.middleware import run_llm_execution_middleware @@ -2208,7 +2672,7 @@ def _perform_api_call(next_api_kwargs): # Terminal — flush buffered retry trace so user sees what happened. agent._flush_status_buffer() agent._emit_status(f"❌ Max retries ({max_retries}) exceeded for invalid responses. Giving up.") - logger.error(f"{agent.log_prefix}Invalid API response after {max_retries} retries.") + logger.error("%sInvalid API response after %d retries.", agent.log_prefix, max_retries) agent._persist_session(messages, conversation_history) _final_response = f"Invalid API response after {max_retries} retries: {_failure_hint}" return { @@ -2223,13 +2687,24 @@ def _perform_api_call(next_api_kwargs): # Backoff before retry — jittered exponential: 5s base, 120s cap wait_time = jittered_backoff(retry_count, base_delay=5.0, max_delay=120.0) agent._buffer_vprint(f"⏳ Retrying in {wait_time:.1f}s ({_failure_hint})...") - logger.warning(f"Invalid API response (retry {retry_count}/{max_retries}): {', '.join(error_details)} | Provider: {provider_name}") + logger.warning("Invalid API response (retry %d/%d): %s | Provider: %s", retry_count, max_retries, ', '.join(error_details), provider_name) # Sleep in small increments to stay responsive to interrupts sleep_end = time.time() + wait_time _backoff_touch_counter = 0 while time.time() < sleep_end: if agent._interrupt_requested: + # A redirect uses the interrupt machinery to cancel + # only the live request. Aborting the retry here + # with clear_interrupt() would DESTROY the pending + # correction and kill the turn with "Operation + # interrupted" — the exact mid-stream steer loss + # users hit when a redirect lands during provider + # backoff. Rebuild from the correction instead, + # mirroring the InterruptedError handler. + if agent.clear_interrupt(preserve_redirect=True): + _retry.restart_with_redirected_messages = True + break agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during retry wait, aborting.", force=True) _interrupt_text = f"Operation interrupted during retry ({_failure_hint}, attempt {retry_count}/{max_retries})." close_interrupted_tool_sequence(messages, _interrupt_text) @@ -2251,6 +2726,8 @@ def _perform_api_call(next_api_kwargs): f"retry backoff ({retry_count}/{max_retries}), " f"{int(sleep_end - time.time())}s remaining" ) + if _retry.restart_with_redirected_messages: + break # rebuild this iteration from the correction continue # Retry the API call agent._turn_received_provider_response = True @@ -2556,10 +3033,27 @@ def _perform_api_call(next_api_kwargs): ) if assistant_message is not None and not _trunc_has_tool_calls: length_continue_retries += 1 - interim_msg = agent._build_assistant_message(assistant_message, finish_reason) - messages.append(interim_msg) - if assistant_message.content: - truncated_response_parts.append(assistant_message.content) + # An EMPTY partial-stream stub (stream dropped + # mid tool-call before any text was delivered) + # must not be appended as an interim assistant + # message: it would serialize as + # {"role": "assistant", "content": ""}, and + # strict providers (Moonshot/Kimi via OpenRouter) + # reject empty assistant content with HTTP 400 + # ("message ... with role 'assistant' must not be + # empty") on the very next replay — permanently + # poisoning the session history. There is no + # partial text to continue from anyway, so only + # the continuation user-message is appended. + _is_empty_partial_stub = ( + getattr(response, "id", "") == PARTIAL_STREAM_STUB_ID + and not getattr(assistant_message, "content", None) + ) + if not _is_empty_partial_stub: + interim_msg = agent._build_assistant_message(assistant_message, finish_reason) + messages.append(interim_msg) + if assistant_message.content: + truncated_response_parts.append(assistant_message.content) if length_continue_retries < 4: _is_partial_stream_stub = ( @@ -2893,7 +3387,12 @@ def _perform_api_call(next_api_kwargs): _cost_delta = (_cost_delta or 0.0) + float(_moa_ref_cost) except (TypeError, ValueError): # pragma: no cover pass - agent._session_db.update_token_counts( + # Enqueued, not written: the background writer + # applies the delta off the turn thread (a cold + # state.db UPDATE here stalled the tool loop for + # up to hundreds of ms per API call). Drained at + # turn finalize via _persist_session. + agent._session_db.queue_token_counts( agent.session_id, input_tokens=canonical_usage.input_tokens, output_tokens=canonical_usage.output_tokens, @@ -2959,6 +3458,12 @@ def _perform_api_call(next_api_kwargs): clear_nous_rate_limit() except Exception: pass + from agent import relay_llm + + relay_llm.complete_logical_call( + api_request_id, + outcome="success", + ) agent._touch_activity(f"API call #{api_call_count} completed") break # Success, exit retry loop @@ -3454,7 +3959,7 @@ def _perform_api_call(next_api_kwargs): agent._buffer_vprint("🔐 Vertex AI token refreshed after 401. Retrying request...") continue if ( - agent.api_mode == "chat_completions" + agent.api_mode in ("chat_completions", "anthropic_messages") and agent.provider == "nous" and status_code == 401 and not _retry.nous_auth_retry_attempted @@ -3486,7 +3991,7 @@ def _perform_api_call(next_api_kwargs): print(f"{agent.log_prefix} • Verify stored credentials: {_dhh}/auth.json") print(f"{agent.log_prefix} • Switch providers temporarily: /model --provider openrouter") if ( - agent.provider == "copilot" + _is_copilot_provider(agent) and status_code == 401 and not _retry.copilot_auth_retry_attempted ): @@ -3726,6 +4231,12 @@ def _perform_api_call(next_api_kwargs): # Check for interrupt before deciding to retry if agent._interrupt_requested: + # Preserve a pending redirect (mid-stream correction): the + # user is steering, not stopping. Rebuild the turn from the + # correction instead of aborting with a dead-end interrupt. + if agent.clear_interrupt(preserve_redirect=True): + _retry.restart_with_redirected_messages = True + break agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during error handling, aborting retries.", force=True) _interrupt_text = f"Operation interrupted: handling API error ({error_type}: {agent._clean_error_message(str(api_error))})." close_interrupted_tool_sequence(messages, _interrupt_text) @@ -3849,9 +4360,11 @@ def _perform_api_call(next_api_kwargs): compression_attempts += 1 if compression_attempts <= max_compression_attempts: original_len = len(messages) + # Option A (LCM issue 441): overhead-aware request size so recovery arms on + # the true request (msgs + tools + system), not the tool-blind message count. messages, active_system_prompt = agent._compress_context( messages, system_message, - approx_tokens=approx_tokens, + approx_tokens=estimate_request_tokens_rough(api_messages, tools=agent.tools or None), task_id=effective_task_id, ) conversation_history = conversation_history_after_compression( @@ -4088,7 +4601,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached for payload-too-large error.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}413 compression failed after {max_compression_attempts} attempts.") + logger.error("%s413 compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Request payload too large: max compression attempts ({max_compression_attempts}) reached." return { @@ -4106,8 +4619,11 @@ def _perform_api_call(next_api_kwargs): original_len = len(messages) original_tokens = estimate_messages_tokens_rough(messages) _overflow_input = messages + # Option A (LCM issue 441): overhead-aware request size so recovery arms on the + # true request (msgs + tools + system), not the tool-blind message count. messages, active_system_prompt = agent._compress_context( - messages, system_message, approx_tokens=approx_tokens, + messages, system_message, + approx_tokens=estimate_request_tokens_rough(api_messages, tools=agent.tools or None), task_id=effective_task_id, ) if messages is _overflow_input and compression_skipped_due_to_lock(agent): @@ -4157,7 +4673,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Payload too large and cannot compress further.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}413 payload too large. Cannot compress further.") + logger.error("%s413 payload too large. Cannot compress further.", agent.log_prefix) agent._persist_session(messages, conversation_history) _final_response = "Request payload too large (413). Cannot compress further." return { @@ -4230,7 +4746,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + logger.error("%sContext compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached." return { @@ -4319,6 +4835,13 @@ def _perform_api_call(next_api_kwargs): provider=agent.provider, api_mode=agent.api_mode, ) + # Persist an explicit provider-reported limit before + # compression/retry. The next request can be rate + # limited, omit usage, or the process can restart; none + # of those should discard metadata the provider already + # confirmed. Keep the probe flags as a best-effort + # post-success retry if this write cannot complete. + save_context_length(agent.model, agent.base_url, new_ctx) # Context probing flags — only set on built-in # compressor (plugin engines manage their own). This # value came from the provider, so it is safe to cache. @@ -4342,7 +4865,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + logger.error("%sContext compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached." return { @@ -4360,8 +4883,13 @@ def _perform_api_call(next_api_kwargs): original_len = len(messages) original_tokens = estimate_messages_tokens_rough(messages) _overflow_input = messages + # Option A (LCM issue 441): pass the OVERHEAD-AWARE request size (msgs + tool + # schemas + system), not the tool-blind message count, so LCM forced-overflow + # recovery arms on the TRUE request that overflowed. See hermes-lcm engine + # _should_force_overflow_recovery. (approx_tokens stays for the status display.) messages, active_system_prompt = agent._compress_context( - messages, system_message, approx_tokens=approx_tokens, + messages, system_message, + approx_tokens=estimate_request_tokens_rough(api_messages, tools=agent.tools or None), task_id=effective_task_id, ) if messages is _overflow_input and compression_skipped_due_to_lock(agent): @@ -4400,7 +4928,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Context length exceeded and cannot compress further.", force=True) agent._vprint(f"{agent.log_prefix} 💡 The conversation has accumulated too much content. Try /new to start fresh, or /compress to manually trigger compression.", force=True) - logger.error(f"{agent.log_prefix}Context length exceeded: {new_tokens:,} tokens. Cannot compress further.") + logger.error("%sContext length exceeded: %s tokens. Cannot compress further.", agent.log_prefix, f"{new_tokens:,}") agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded ({new_tokens:,} tokens). Cannot compress further." return { @@ -4486,6 +5014,32 @@ def _perform_api_call(next_api_kwargs): ) and not is_context_length_error if is_client_error: + # Copilot self-heal BEFORE fallback: a stale/degraded + # credential surfaces as a 400 + # ``model_not_available_for_integrator`` / + # ``model_not_supported`` (not a clean 401), so the 401 + # refresh path above never fired. Force a fresh token + # exchange + client rebuild and retry once on the SAME + # provider — a fresh 437-char API token routes to the + # correct integrator and the model becomes available again. + # Single-shot guard prevents looping on a genuinely + # unavailable model. Copilot-scoped so other providers' + # real 400s are untouched. + if ( + _is_copilot_provider(agent) + and not _retry.copilot_stale_cred_retry_attempted + and _is_stale_copilot_credential_error( + status_code, str(getattr(api_error, "message", "") or api_error) + ) + ): + _retry.copilot_stale_cred_retry_attempted = True + if agent._try_recover_stale_copilot_credential(): + agent._buffer_vprint( + "🔐 Copilot credential re-exchanged after " + "model_not_available 400. Retrying request..." + ) + retry_count = 0 + continue # Try fallback before aborting — a different provider may # not have the same issue (rate limit, auth, etc.). Only # announce the attempt when a fallback chain actually @@ -4640,7 +5194,7 @@ def _perform_api_call(next_api_kwargs): f"{agent.log_prefix} for localhost, or add the server's cert to your trust store.", force=True, ) - logger.error(f"{agent.log_prefix}Non-retryable client error: {api_error}") + logger.error("%sNon-retryable client error: %s", agent.log_prefix, api_error) # Skip session persistence when the error is likely # context-overflow related (status 400 + large session). # Persisting the failed user message would make the @@ -4962,6 +5516,12 @@ def _perform_api_call(next_api_kwargs): _backoff_touch_counter = 0 while time.time() < sleep_end: if agent._interrupt_requested: + # Same preserve-redirect rule as the retry-wait above: + # a steering correction must survive backoff, not die + # as "Operation interrupted". + if agent.clear_interrupt(preserve_redirect=True): + _retry.restart_with_redirected_messages = True + break agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during retry wait, aborting.", force=True) _interrupt_text = f"Operation interrupted: retrying API call after error (retry {retry_count}/{max_retries})." close_interrupted_tool_sequence(messages, _interrupt_text) @@ -4983,6 +5543,11 @@ def _perform_api_call(next_api_kwargs): f"error retry backoff ({retry_count}/{max_retries}), " f"{int(sleep_end - time.time())}s remaining" ) + if _retry.restart_with_redirected_messages: + # Leave the retry loop — the check right below rebuilds this + # iteration from the correction instead of re-firing the + # stale request. + break if _retry.restart_with_redirected_messages: # The cancelled request produced no valid assistant item. Reuse the @@ -5086,7 +5651,7 @@ def _perform_api_call(next_api_kwargs): assistant_message.content = str(raw) try: - from hermes_cli.plugins import ( + from hermes_cli.lifecycle import ( has_hook, invoke_hook as _invoke_hook, ) @@ -5328,6 +5893,14 @@ def _perform_api_call(next_api_kwargs): args_preview = raw_args[:200] if isinstance(raw_args, str) else repr(raw_args)[:200] logging.debug("Tool call: %s with args: %s...", tc.function.name, args_preview) + # Uniquify duplicate tool-call ids BEFORE any downstream + # consumer (validation error paths, dispatch, history build, + # Responses item-id derivation). Models that reuse one id for + # different calls in a batch otherwise lose the later call's + # result: the pre-API sanitizer keeps only the first + # call/result pair per id. See _uniquify_tool_call_ids. + agent._uniquify_tool_call_ids(assistant_message.tool_calls) + # Validate tool call names - detect model hallucinations # Repair mismatched tool names before validating for tc in assistant_message.tool_calls: @@ -5616,6 +6189,10 @@ def _perform_api_call(next_api_kwargs): # flag so it can fire again if the model goes empty on # a LATER tool round. agent._post_tool_empty_retried = False + # A landed tool call means any earlier dropped-tool-call stall + # was recovered — refresh that budget too so it guards each + # stall independently rather than capping the whole run. + agent._dropped_toolcall_retries = 0 previous_msg = messages[-1] if messages else None current_interim_visible = agent._interim_assistant_visible_text(assistant_msg) @@ -5632,8 +6209,6 @@ def _perform_api_call(next_api_kwargs): and previous_interim_visible == current_interim_visible ) messages.append(assistant_msg) - if not duplicate_previous_interim: - agent._emit_interim_assistant_message(assistant_msg) # Mixed batch: error-result the invalid calls and strip them # from the execution set. The assistant message above keeps @@ -5655,13 +6230,17 @@ def _perform_api_call(next_api_kwargs): if tc.function.name in agent.valid_tool_names ] + _tool_turn_persisted = None try: # Persist the assistant tool-call turn before any tool # side effects run. If a destructive tool restarts or # terminates Hermes mid-turn, resume logic still sees the # exact tool-call block that already executed. - agent._flush_messages_to_session_db(messages, conversation_history) + _tool_turn_persisted = agent._flush_messages_to_session_db( + messages, conversation_history + ) except Exception as exc: + _tool_turn_persisted = False logger.warning( "Incremental tool-call persistence failed before execution " "(session=%s): %s", @@ -5669,6 +6248,22 @@ def _perform_api_call(next_api_kwargs): exc, ) + if _tool_turn_persisted is False: + # The canonical append failed. Do not project the row or + # run side-effecting tools from state that exists only in + # this process. Breaking also avoids retrying the same + # unpersisted turn until the iteration budget is exhausted. + _turn_exit_reason = "session_persistence_failed" + final_response = "" + failed = True + break + + # A UI must never observe an assistant/tool-call row that is + # still only an ephemeral in-memory projection. Emit interim + # commentary only after the canonical SessionDB append above. + if not duplicate_previous_interim: + agent._emit_interim_assistant_message(assistant_msg) + # Close any open streaming display (response box, reasoning # box) before tool execution begins. Intermediate turns may # have streamed early content that opened the response box; @@ -5683,6 +6278,15 @@ def _perform_api_call(next_api_kwargs): agent._execute_tool_calls(assistant_message, messages, effective_task_id, api_call_count) + if getattr(agent, "_incremental_persistence_failed", False): + # A tool result could not be made canonical. Do not send + # the in-memory result back to the model or project any + # later events from this turn. + _turn_exit_reason = "session_persistence_failed" + final_response = "" + failed = True + break + if agent._tool_guardrail_halt_decision is not None: decision = agent._tool_guardrail_halt_decision _turn_exit_reason = "guardrail_halt" @@ -5779,9 +6383,12 @@ def _perform_api_call(next_api_kwargs): _clear_warn() agent._safe_print(" ⟳ compacting context…") _post_tool_input = messages + # Route the overhead-aware _real_tokens (computed above) into compression, not + # the bare last_prompt_tokens — which is 0 in the no-usage fallback, hiding the + # true request size from the engine's overflow guard (upstream PR #77169 review). messages, active_system_prompt = agent._compress_context( messages, system_message, - approx_tokens=agent.context_compressor.last_prompt_tokens, + approx_tokens=_real_tokens, task_id=effective_task_id, ) if ( @@ -5864,11 +6471,26 @@ def _perform_api_call(next_api_kwargs): # Save session log incrementally (so progress is visible even if interrupted) agent._session_messages = messages + # Touch activity before continuing so the gateway's + # inactivity monitor never sees a stale timestamp + # between tool completion and the start of the next + # API call. Without this, a tool-call result (which + # takes ~0s to process) followed by slow post-tool + # processing (compression, persist) and a slow + # follow-up API call can exceed the gateway inactivity + # timeout (HERMES_AGENT_TIMEOUT, default 1800s) and the + # gateway kills the session before the next activity + # touch fires (#69559, #69131). + agent._touch_activity(f"tool results posted, continuing iteration #{api_call_count}") # Continue loop for next response continue else: - # No tool calls - this is the final response + # No tool calls - this is the final response. + # (Dropped tool-call recovery — finish_reason=="tool_calls" with + # an empty tool_calls array — is handled at the finalization + # chokepoint below, after final_msg is built, so it catches + # every path that reaches turn finalization, not just this one.) final_response = assistant_message.content or "" # Fix: unmute output when entering the no-tool-call branch @@ -6053,15 +6675,47 @@ def _perform_api_call(next_api_kwargs): ) if _truly_empty and (not _has_structured or _prefill_exhausted) and agent._empty_content_retries < 3: agent._empty_content_retries += 1 + wait_time = jittered_backoff( + agent._empty_content_retries, + base_delay=5.0, + max_delay=60.0, + ) logger.warning( "Empty response (no content or reasoning) — " - "retry %d/3 (model=%s)", - agent._empty_content_retries, agent.model, + "retry %d/3 in %.1fs (model=%s)", + agent._empty_content_retries, wait_time, agent.model, ) agent._buffer_status( f"⚠️ Empty response from model — retrying " - f"({agent._empty_content_retries}/3)" + f"({agent._empty_content_retries}/3) in {wait_time:.0f}s" ) + # Sleep in small increments to stay responsive to interrupts + sleep_end = time.time() + wait_time + _backoff_touch_counter = 0 + while time.time() < sleep_end: + if agent._interrupt_requested: + agent._vprint(f"{agent.log_prefix}⚡ Interrupt detected during empty-response retry wait, aborting.", force=True) + _interrupt_text = ( + f"Operation interrupted: retrying empty response from model " + f"(retry {agent._empty_content_retries}/3)." + ) + close_interrupted_tool_sequence(messages, _interrupt_text) + agent._persist_session(messages, conversation_history) + agent.clear_interrupt() + return { + "final_response": _interrupt_text, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "interrupted": True, + } + time.sleep(0.2) + _backoff_touch_counter += 1 + if _backoff_touch_counter % 150 == 0: # 150 × 0.2s = 30s + agent._touch_activity( + f"empty response retry backoff ({agent._empty_content_retries}/3), " + f"{int(sleep_end - time.time())}s remaining" + ) continue # ── Exhausted retries — try fallback provider ── @@ -6140,7 +6794,28 @@ def _perform_api_call(next_api_kwargs): ". No fallback providers configured.") ) - final_response = "(empty)" + # Deliver a labeled reasoning excerpt instead of a bare + # "(empty)" when the model DID think but never produced + # visible text. This is delivery-only: the persisted + # assistant message above keeps the "(empty)" sentinel + # (its replay semantics prevent empty-response loops), + # and raw chain-of-thought is never promoted to a normal + # answer earlier in the ladder — prefill continuation, + # empty-content retries, and provider fallback all run + # first. Only at this terminal, where the alternative is + # returning nothing, is showing the model's own reasoning + # (clearly labeled as such) strictly more useful. + # Idea credit: PR #48795 (@ligl0325). + if reasoning_text: + final_response = ( + "⚠️ The model produced only internal reasoning and " + "no final answer, despite retries" + + (" and fallback" if agent._fallback_chain else "") + + ". Its last reasoning, which may contain the " + "answer:\n\n" + reasoning_preview + ) + else: + final_response = "(empty)" break # Reset retry counter/signature on successful content @@ -6200,6 +6875,64 @@ def _perform_api_call(next_api_kwargs): final_msg = agent._build_assistant_message(assistant_message, finish_reason) + # ── Dropped tool-call recovery (copilot/Claude) ──────── + # Some providers (observed: claude-opus-4.8 / claude-sonnet-4.5 + # on GitHub Copilot, ~2026-07) return finish_reason="tool_calls" + # while the parsed tool_calls array is empty — the model + # signalled it wanted to act but the payload shipped no call. + # Reaching finalization with that mismatch means the turn is + # about to end with the task unstarted (the narration, which may + # be in content or only in the reasoning field, gets treated as + # the final answer). Re-prompt (bounded to 3 CONSECUTIVE stalls; + # the budget resets after any successful tool round) to make the + # model emit the call instead of exiting. finish_reason="stop" + # text finishes never enter this guard. + if ( + finish_reason == "tool_calls" + and not assistant_message.tool_calls + and getattr(agent, "_dropped_toolcall_retries", 0) < 3 + ): + agent._dropped_toolcall_retries = getattr(agent, "_dropped_toolcall_retries", 0) + 1 + logger.warning( + "finish_reason=tool_calls with empty tool_calls array " + "(narration only) — re-prompting to emit the call " + "(retry %d/3, model=%s provider=%s)", + agent._dropped_toolcall_retries, agent.model, agent.provider, + ) + agent._emit_status( + "↻ Model signaled a tool call but sent none — " + f"re-prompting ({agent._dropped_toolcall_retries}/3)" + ) + # Both halves of the re-prompt pair are ephemeral recovery + # scaffolding (mirrors the empty-response nudge pattern): + # the interim narration-only assistant turn exists solely to + # keep role alternation valid for the nudge, and the nudge + # exists solely to drive the retry. Flag both so the + # persistence layer never writes them to the durable + # transcript and the finalization pop below can strip an + # unanswered tail pair. A recovered (answered) pair stays + # buried mid-list in live memory but is skipped by the + # flush regardless of position. + final_msg["_dropped_toolcall_nudge"] = True + messages.append(final_msg) + messages.append({ + "role": "user", + "content": ( + "Your previous turn indicated a tool call but none was " + "included. Do not narrate a plan or restate intent — issue " + "the actual tool call now to continue the task." + ), + "_dropped_toolcall_nudge": True, + }) + agent._session_messages = messages + final_response = None + continue + + # Reached finalization without the dropped-tool-call mismatch — + # a genuine turn end. Clear the consecutive-stall budget so the + # next turn starts fresh. + agent._dropped_toolcall_retries = 0 + # Pop thinking-only prefill and empty-response retry # scaffolding before appending either a final response or a # verification-stop follow-up. These internal turns are only @@ -6212,6 +6945,7 @@ def _perform_api_call(next_api_kwargs): messages[-1].get("_thinking_prefill") or messages[-1].get("_empty_recovery_synthetic") or messages[-1].get("_empty_terminal_sentinel") + or messages[-1].get("_dropped_toolcall_nudge") ) ): messages.pop() @@ -6285,7 +7019,8 @@ def _perform_api_call(next_api_kwargs): _attempt = getattr(agent, "_pre_verify_nudges", 0) try: from agent.verify_hooks import max_verify_nudges - from hermes_cli.plugins import get_pre_verify_continue_message, has_hook + from hermes_cli.lifecycle import has_hook + from hermes_cli.plugins import get_pre_verify_continue_message if _edited and has_hook("pre_verify") and _attempt < max_verify_nudges(): # Posture is fixed for the session — resolve once + cache. diff --git a/agent/copilot_acp_client.py b/agent/copilot_acp_client.py index 662facc2dfee..9cbdcd349446 100644 --- a/agent/copilot_acp_client.py +++ b/agent/copilot_acp_client.py @@ -512,7 +512,7 @@ def _run_prompt(self, prompt_text: str, *, timeout_seconds: float) -> tuple[str, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, + text=True, encoding='utf-8', errors='replace', bufsize=1, cwd=self._acp_cwd, env=_build_subprocess_env(), @@ -708,7 +708,7 @@ def _handle_server_message( if block_error: raise PermissionError(block_error) try: - content = path.read_text() + content = path.read_text(encoding="utf-8") except FileNotFoundError: content = "" line = params.get("line") @@ -736,7 +736,7 @@ def _handle_server_message( if denied: raise PermissionError(denied) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(str(params.get("content") or "")) + path.write_text(str(params.get("content") or ""), encoding="utf-8") response = { "jsonrpc": "2.0", "id": message_id, diff --git a/agent/credential_pool.py b/agent/credential_pool.py index d5d652ad7414..1022665087ce 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -28,10 +28,13 @@ _auth_store_lock, _codex_access_token_is_expiring, _decode_jwt_claims, + _global_auth_file_path, _load_auth_store, _load_provider_state, + _load_provider_state_with_source, _resolve_kimi_base_url, _resolve_zai_base_url, + _same_path, _save_auth_store, _save_provider_state, _store_provider_state, @@ -585,7 +588,12 @@ def __init__(self, provider: str, entries: List[PooledCredential]): self._entries = sorted(entries, key=lambda entry: entry.priority) self._current_id: Optional[str] = None self._strategy = get_pool_strategy(provider) - self._lock = threading.Lock() + # RLock: the mutation primitives below (_replace_entry/_persist) + # self-acquire this lock so the DEFERRED single-use-token refresh + # path (which runs network I/O outside the lock by design) still + # serializes its pool mutations. In-lock callers re-acquire + # reentrantly at negligible cost. + self._lock = threading.RLock() self._active_leases: Dict[str, int] = {} self._max_concurrent = DEFAULT_MAX_CONCURRENT_PER_CREDENTIAL # Monotonic timestamp of the last "no available entries" log, used to @@ -594,6 +602,14 @@ def __init__(self, provider: str, entries: List[PooledCredential]): # Re-armed to None on every successful selection so a recover→re-exhaust # transition logs promptly instead of being swallowed by a stale window. self._last_no_entries_log_at: Optional[float] = None + # #70401: consecutive mark_exhausted_and_rotate() calls whose supplied + # credential identity matched no pool entry (OAuth wrappers whose + # runtime key rotates, entries pruned by another process, ...). These + # rotations mark nothing exhausted, so without a cap the pool can + # never converge to "no available entries" and the caller's 401 retry + # loop runs unbounded and non-interruptible. Reset whenever a real + # entry is identified or an escape path returns None. + self._unmatched_rotation_streak: int = 0 def has_credentials(self) -> bool: with self._lock: @@ -607,7 +623,36 @@ def has_available(self) -> bool: # otherwise a status probe here can race a concurrent ``select`` / # rotation and tear ``self._entries`` or double-write auth.json. with self._lock: - return bool(self._available_entries()) + available, _pending = self._available_entries() + return bool(available) + + def next_available_at(self) -> Optional[float]: + """Earliest epoch time (seconds) any entry re-enters rotation. + + Returns ``None`` when at least one entry is available right now, or + when no exhausted entry carries a usable recovery time (empty pool, + or only ``STATUS_DEAD`` entries, which never re-enter via TTL). + Callers must treat ``None`` as "no wait information", not + "unavailable". + + Like :meth:`has_available`, expired cooldowns are left uncleared + (``clear_expired=False``); the only writes are the same + re-auth/token sync paths ``has_available`` already performs — which + is exactly why this must run under ``self._lock`` like every other + ``_available_entries`` caller (see the comment on ``has_available``). + """ + with self._lock: + available, _pending = self._available_entries() + if available: + return None + candidates: List[float] = [] + for entry in self._entries: + if entry.last_status != STATUS_EXHAUSTED: + continue + until = _exhausted_until(entry) + if until is not None: + candidates.append(until) + return min(candidates) if candidates else None def entries(self) -> List[PooledCredential]: with self._lock: @@ -622,19 +667,50 @@ def current(self) -> Optional[PooledCredential]: with self._lock: return self._current_unlocked() + def entry_id_for_api_key(self, api_key_hint: Any = None) -> Optional[str]: + """Return the stable id for the runtime credential in use. + + Prefer the current selection when it still supplies ``api_key_hint``. + If the cursor was cleared, fall back to an unambiguous key match. + """ + with self._lock: + current = self._current_unlocked() + if current is not None and ( + api_key_hint is None + or current.runtime_api_key == api_key_hint + ): + return current.id + if api_key_hint is None: + return None + matches = [ + entry + for entry in self._entries + if entry.runtime_api_key == api_key_hint + ] + return matches[0].id if len(matches) == 1 else None + def _replace_entry(self, old: PooledCredential, new: PooledCredential) -> None: - """Swap an entry in-place by id, preserving sort order.""" - for idx, entry in enumerate(self._entries): - if entry.id == old.id: - self._entries[idx] = new - return + """Swap an entry in-place by id, preserving sort order. + + Self-locking (RLock) so the deferred refresh path — which + deliberately runs outside the pool lock — cannot tear + ``self._entries`` against a concurrent select()/rotation. + """ + with self._lock: + for idx, entry in enumerate(self._entries): + if entry.id == old.id: + self._entries[idx] = new + return def _persist(self, *, removed_ids: Optional[List[str]] = None) -> None: - write_credential_pool( - self.provider, - [entry.to_dict() for entry in self._entries], - removed_ids=removed_ids, - ) + # Self-locking (RLock): snapshotting self._entries must not race a + # concurrent rotation when called from the deferred refresh path. + with self._lock: + write_credential_pool( + self.provider, + [entry.to_dict() for entry in self._entries], + removed_ids=removed_ids, + ) def _is_terminal_auth_failure( self, @@ -762,7 +838,7 @@ def _sync_codex_entry_from_auth_store(self, entry: PooledCredential) -> PooledCr device_code-sourced entries; env/API-key-sourced entries have no auth.json shadow to sync from. """ - if self.provider != "openai-codex" or entry.source != "device_code": + if self.provider != "openai-codex" or entry.source not in ("device_code", "manual:device_code"): return entry try: with _auth_store_lock(): @@ -778,19 +854,43 @@ def _sync_codex_entry_from_auth_store(self, entry: PooledCredential) -> PooledCr # Adopt auth.json tokens when either side differs. Codex refresh # tokens are single-use too, so a fresh refresh_token from # another process means our entry's pair is consumed/stale. + # + # Also adopt when the store has a refresh_token but no + # access_token — another process may have rotated the pair + # and the store entry's access_token was already consumed; + # the important signal is the refresh_token difference. entry_access = entry.access_token or "" entry_refresh = entry.refresh_token or "" + should_adopt = False if store_access and ( store_access != entry_access or (store_refresh and store_refresh != entry_refresh) ): + should_adopt = True + elif ( + store_refresh + and store_refresh != entry_refresh + and not store_access + ): + # Store has only a refresh_token (no access_token) — + # another process rotated the pair. Adopt the + # refresh_token so we don't replay the consumed one. + logger.info( + "Pool entry %s: auth.json has newer refresh_token " + "but no access_token; adopting refresh_token to " + "avoid replaying consumed token", + entry.id, + ) + should_adopt = True + + if should_adopt: logger.debug( "Pool entry %s: syncing Codex tokens from auth.json " "(refreshed by another process)", entry.id, ) field_updates: Dict[str, Any] = { - "access_token": store_access, + "access_token": store_access or entry.access_token, "refresh_token": store_refresh or entry.refresh_token, "last_status": None, "last_status_at": None, @@ -1009,32 +1109,58 @@ def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None try: with _auth_store_lock(): auth_store = _load_auth_store() - # Decide BEFORE writing whether this profile is reading the - # grant from the global root (no own providers. block) vs. - # genuinely shadowing it. A pool refresh rotates single-use - # OAuth refresh tokens, so a profile that resolved the grant - # from root MUST write the rotated chain back to root too — - # otherwise root keeps a revoked refresh token and every other - # profile reading the stale root grant dies with - # refresh_token_reused / invalid_grant once its access token - # expires. This mirrors the xAI write-through in - # hermes_cli.auth._save_xai_oauth_tokens (#43589); the pool - # refresh path is the Codex/xAI analog reported in #48415. _wt_provider_id = { "nous": "nous", "openai-codex": "openai-codex", "xai-oauth": "xai-oauth", }.get(self.provider) - write_through_to_root = bool(_wt_provider_id) and not ( - isinstance(auth_store.get("providers"), dict) - and isinstance( - auth_store["providers"].get(_wt_provider_id), dict - ) - ) + # Resolve state and track which store it came from — the + # source path tells us whether this profile genuinely owns + # its provider block or is reading from the global root. + # #74339: the old key-presence check decided write-through + # on whether the profile had ``providers.`` BEFORE the + # save — correct for the first refresh but self-sealing + # because ``_store_provider_state`` unconditionally creates + # that key inside the same function. Once the profile has + # the key, every subsequent refresh silently disables the + # root write-through and root keeps a revoked refresh token. + # + # Fix: use ``_load_provider_state_with_source`` to learn + # where the state was resolved from. When the grant was + # resolved from the global root, write back *only* to root + # and skip ``_store_provider_state`` for the profile so the + # profile does not accrue a shadowing ``providers.`` + # key that blocks both the root fallback and the write-through + # on subsequent calls. if self.provider == "nous": - state = _load_provider_state(auth_store, "nous") + state, source_path = _load_provider_state_with_source( + auth_store, "nous" + ) if state is None: return + elif self.provider == "openai-codex": + state, source_path = _load_provider_state_with_source( + auth_store, "openai-codex" + ) + if not isinstance(state, dict): + return + elif self.provider == "xai-oauth": + state, source_path = _load_provider_state_with_source( + auth_store, "xai-oauth" + ) + if not isinstance(state, dict): + return + else: + return + + global_root = _global_auth_file_path() + is_from_root = bool( + source_path is not None + and global_root is not None + and _same_path(source_path, global_root) + ) + + if self.provider == "nous": state["access_token"] = entry.access_token if entry.refresh_token: state["refresh_token"] = entry.refresh_token @@ -1052,12 +1178,8 @@ def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None state[extra_key] = val if entry.inference_base_url: state["inference_base_url"] = entry.inference_base_url - _store_provider_state(auth_store, "nous", state, set_active=False) elif self.provider == "openai-codex": - state = _load_provider_state(auth_store, "openai-codex") - if not isinstance(state, dict): - return tokens = state.get("tokens") if not isinstance(tokens, dict): return @@ -1066,12 +1188,8 @@ def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None tokens["refresh_token"] = entry.refresh_token if entry.last_refresh: state["last_refresh"] = entry.last_refresh - _store_provider_state(auth_store, "openai-codex", state, set_active=False) elif self.provider == "xai-oauth": - state = _load_provider_state(auth_store, "xai-oauth") - if not isinstance(state, dict): - return tokens = state.get("tokens") if not isinstance(tokens, dict): return @@ -1080,16 +1198,26 @@ def _sync_device_code_entry_to_auth_store(self, entry: PooledCredential) -> None tokens["refresh_token"] = entry.refresh_token if entry.last_refresh: state["last_refresh"] = entry.last_refresh - _store_provider_state(auth_store, "xai-oauth", state, set_active=False) - else: - return - - _save_auth_store(auth_store) - if write_through_to_root and _wt_provider_id: + if is_from_root and _wt_provider_id: + # Grant was resolved from root — write back to root + # only. Do NOT call _store_provider_state on the + # profile auth_store (it would create a shadowing + # providers. key that disables write-through on + # the next refresh — #74339). + # _load_provider_state has root fallback, so the + # profile can always read fresh tokens from root + # without needing its own providers block. _write_through_provider_state_to_global_root( _wt_provider_id, state ) + else: + # Profile genuinely owns this provider — write to + # the profile store as normal. + _store_provider_state( + auth_store, self.provider, state, set_active=False + ) + _save_auth_store(auth_store) except Exception as exc: logger.debug("Failed to sync %s pool entry back to auth store: %s", self.provider, exc) @@ -1561,20 +1689,61 @@ def _entry_needs_refresh(self, entry: PooledCredential) -> bool: return False def select(self) -> Optional[PooledCredential]: + entry, pending_refresh = self._select_under_lock() + if pending_refresh: + self._refresh_pending_entries(pending_refresh) + if entry is not None: + self._unmatched_rotation_streak = 0 + return entry + # If no entry was available but we just refreshed some, re-select + # now that the refreshed entries are back in the pool. + if pending_refresh: + entry, _ = self._select_under_lock() + if entry is not None: + self._unmatched_rotation_streak = 0 + return entry + + def _select_under_lock(self) -> Tuple[Optional[PooledCredential], List[tuple]]: + """Run selection under the lock, returning entry + pending refreshes.""" with self._lock: return self._select_unlocked() - def _available_entries(self, *, clear_expired: bool = False, refresh: bool = False) -> List[PooledCredential]: - """Return entries not currently in exhaustion cooldown. + def _refresh_pending_entries(self, pending: List[tuple]) -> None: + """Refresh deferred single-use-token entries outside the lock. + + Each entry is refreshed under the cross-process ``_auth_store_lock`` + (which can block for 20+ seconds) and then merged into the pool. + On failure the entry is silently skipped. + """ + for entry, sync_fn in pending: + # _refresh_entry already merges the refreshed entry into the + # pool internally (its mutation primitives are self-locking), + # so no second _replace_entry is needed here. + self._refresh_entry(entry, force=False) + + def _available_entries( + self, *, clear_expired: bool = False, refresh: bool = False, + ) -> Tuple[List[PooledCredential], List[tuple]]: + """Return (available, pending_refresh) for entries not in cooldown. When *clear_expired* is True, entries whose cooldown has elapsed are reset to STATUS_OK and persisted. When *refresh* is True, entries that need a token refresh are refreshed (skipped on failure). + + Single-use-token refreshes (openai-codex, xai-oauth) are returned as + *pending_refresh* tuples so the caller can execute them outside the + lock, avoiding stalling all pool consumers during cross-process flock + acquisition + OAuth network I/O. """ now = time.time() cleared_any = False entries_to_prune: List[str] = [] available: List[PooledCredential] = [] + # Entries that need an OAuth refresh via a single-use token provider + # (openai-codex, xai-oauth). These require a cross-process file lock + # that can block for 20+ seconds. We collect them under self._lock + # and refresh outside the lock to avoid stalling all pool consumers. + pending_refresh: List[tuple] = [] # (entry, sync_entry_fn) for entry in self._entries: # Borrowed credentials persist as metadata-only references and are # hydrated from their live source on load. A stale duplicate row @@ -1683,6 +1852,16 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal entry = cleared cleared_any = True if refresh and self._entry_needs_refresh(entry): + if self.provider in ("openai-codex", "xai-oauth"): + # Defer single-use-token refresh to avoid holding the + # threading lock during cross-process flock + network I/O. + sync_fn = ( + self._sync_codex_entry_from_auth_store + if self.provider == "openai-codex" + else self._sync_xai_oauth_entry_from_pool_store + ) + pending_refresh.append((entry, sync_fn)) + continue refreshed = self._refresh_entry(entry, force=False) if refreshed is None: continue @@ -1693,7 +1872,7 @@ def _available_entries(self, *, clear_expired: bool = False, refresh: bool = Fal self._entries = [e for e in self._entries if e.id not in pruned_ids] if cleared_any: self._persist(removed_ids=entries_to_prune) - return available + return available, pending_refresh def _log_no_available_entries(self) -> None: """Emit the empty-pool INFO line at most once per throttle window. @@ -1709,12 +1888,17 @@ def _log_no_available_entries(self) -> None: self._last_no_entries_log_at = now logger.info("credential pool: no available entries (all exhausted or empty)") - def _select_unlocked(self, *, refresh: bool = True) -> Optional[PooledCredential]: - available = self._available_entries(clear_expired=True, refresh=refresh) + def _select_unlocked(self, *, refresh: bool = True) -> Tuple[Optional[PooledCredential], List[tuple]]: + """Select the best available credential entry. + + Returns ``(entry, pending_refresh)`` where *pending_refresh* contains + single-use-token entries that must be refreshed outside the lock. + """ + available, pending_refresh = self._available_entries(clear_expired=True, refresh=refresh) if not available: self._current_id = None self._log_no_available_entries() - return None + return None, pending_refresh # A successful selection means the pool recovered; re-arm the throttle # so a later re-exhaustion logs immediately rather than being silenced @@ -1724,7 +1908,7 @@ def _select_unlocked(self, *, refresh: bool = True) -> Optional[PooledCredential if self._strategy == STRATEGY_RANDOM: entry = random.choice(available) self._current_id = entry.id - return entry + return entry, pending_refresh if self._strategy == STRATEGY_LEAST_USED and len(available) > 1: entry = min(available, key=lambda e: e.request_count) @@ -1732,7 +1916,7 @@ def _select_unlocked(self, *, refresh: bool = True) -> Optional[PooledCredential updated = replace(entry, request_count=entry.request_count + 1) self._replace_entry(entry, updated) self._current_id = entry.id - return updated + return updated, pending_refresh if self._strategy == STRATEGY_ROUND_ROBIN and len(available) > 1: entry = available[0] @@ -1741,11 +1925,11 @@ def _select_unlocked(self, *, refresh: bool = True) -> Optional[PooledCredential self._entries = [replace(candidate, priority=idx) for idx, candidate in enumerate(rotated)] self._persist() self._current_id = entry.id - return self._current_unlocked() or entry + return self._current_unlocked() or entry, pending_refresh entry = available[0] self._current_id = entry.id - return entry + return entry, pending_refresh def peek(self) -> Optional[PooledCredential]: # Single lock acquisition for the whole read; call the unlocked @@ -1754,7 +1938,7 @@ def peek(self) -> Optional[PooledCredential]: current = self._current_unlocked() if current is not None: return current - available = self._available_entries() + available, _pending = self._available_entries() return available[0] if available else None def mark_exhausted_and_rotate( @@ -1763,10 +1947,17 @@ def mark_exhausted_and_rotate( status_code: Optional[int], error_context: Optional[Dict[str, Any]] = None, api_key_hint: Optional[str] = None, + credential_id: Optional[str] = None, ) -> Optional[PooledCredential]: with self._lock: entry = None - if api_key_hint: + identity_supplied = bool(credential_id or api_key_hint) + if credential_id: + entry = next( + (e for e in self._entries if e.id == credential_id), + None, + ) + if entry is None and api_key_hint: # Prefer the specific entry whose API key matches the one that # actually failed. When this pool was freshly loaded from disk # (another process already rotated), current() is None and @@ -1775,22 +1966,63 @@ def mark_exhausted_and_rotate( (e for e in self._entries if e.runtime_api_key == api_key_hint), None, ) - if entry is None: - # The failed key is identifiable but matches no entry - # (rotated away, or a wrapper whose runtime key differs). - # Falling through to current()/_select_unlocked() would - # mark an INNOCENT healthy key exhausted for the full - # cooldown TTL. Don't guess — just hand back a fresh - # selection so the caller can retry. - logger.info( - "credential pool: failed key hint matched no %s entry; " - "rotating without marking any credential exhausted", + if entry is None and identity_supplied: + # The failed credential is identifiable but matches no entry + # (rotated away, or a wrapper whose runtime key differs). + # Falling through to current()/_select_unlocked() would mark an + # innocent healthy key exhausted for the full cooldown TTL. + # + # #70401: this branch must still be BOUNDED. With OAuth-token + # auth the upstream 401's key hint never matches any entry's + # ``runtime_api_key``, so every retry lands here, nothing is + # ever marked exhausted, and the pool can never reach the + # "no available entries" state — the caller retries the same + # dead token forever (~6/sec, starving the event loop so chat + # interrupts are never processed). The single-entry case + # below already escapes; multi-entry pools could still + # ping-pong A→B→A indefinitely without marking anything. + # Cap consecutive no-mark rotations at one full lap of the + # available entries: past that, every candidate has been + # handed back at least once without recovery, so stop + # guessing and surface the error (no cooldown is written for + # anybody — healthy keys stay available for the next turn). + self._unmatched_rotation_streak += 1 + available_count, _ = self._available_entries() + available_count = len(available_count) + if self._unmatched_rotation_streak > max(available_count, 1): + logger.warning( + "credential pool: failed credential identity matched no " + "%s entry for %d consecutive rotations (pool size %d) — " + "surfacing the error instead of rotating again", self.provider, + self._unmatched_rotation_streak, + available_count, ) + self._unmatched_rotation_streak = 0 + self._current_id = None + return None + logger.info( + "credential pool: failed credential identity matched no %s " + "entry; rotating without marking any credential exhausted", + self.provider, + ) + self._current_id = None + next_entry, _pending = self._select_unlocked(refresh=False) + avail, _ = self._available_entries() + if next_entry is not None and len(avail) == 1: + # A single-entry pool cannot rotate. Returning its only + # entry reports a successful recovery without changing + # the credential, so the caller retries the same 401 + # indefinitely. Let fallback/error propagation proceed. + self._unmatched_rotation_streak = 0 self._current_id = None - return self._select_unlocked() + return None + return next_entry + # A real entry was identified — any prior unmatched-rotation + # streak is stale (this mark WILL advance pool state). + self._unmatched_rotation_streak = 0 if entry is None: - entry = self._current_unlocked() or self._select_unlocked() + entry = self._current_unlocked() or self._select_unlocked(refresh=False)[0] if entry is None: return None _label = entry.label or entry.id[:8] @@ -1806,12 +2038,13 @@ def mark_exhausted_and_rotate( # disconnects (a ~2.5min hang with no error surfaced to the user). # Mark every entry sharing the failed key so the pool can reach the # "no available entries" state and let the error propagate. - if api_key_hint: + failed_runtime_key = getattr(entry, "runtime_api_key", None) + if identity_supplied and failed_runtime_key: siblings_marked = False for sibling in self._entries: if sibling.id == entry.id: continue - if sibling.runtime_api_key == api_key_hint: + if sibling.runtime_api_key == failed_runtime_key: self._mark_exhausted( sibling, status_code, error_context, persist=False ) @@ -1834,7 +2067,7 @@ def mark_exhausted_and_rotate( _label, status_code, ) self._current_id = None - next_entry = self._select_unlocked() + next_entry, _pending = self._select_unlocked(refresh=False) if next_entry: _next_label = next_entry.label or next_entry.id[:8] logger.info("credential pool: rotated to %s", _next_label) @@ -1848,15 +2081,24 @@ def acquire_lease(self, credential_id: Optional[str] = None) -> Optional[str]: a stable tie-breaker. When every credential is already at the soft cap, still return the least-leased one instead of blocking. """ + chosen_id, pending_refresh = self._acquire_lease_under_lock(credential_id) + if pending_refresh: + self._refresh_pending_entries(pending_refresh) + return chosen_id + + def _acquire_lease_under_lock( + self, credential_id: Optional[str], + ) -> Tuple[Optional[str], List[tuple]]: + """Run lease acquisition under the lock, returning id + pending refreshes.""" with self._lock: if credential_id: self._active_leases[credential_id] = self._active_leases.get(credential_id, 0) + 1 self._current_id = credential_id - return credential_id + return credential_id, [] - available = self._available_entries(clear_expired=True, refresh=True) + available, pending_refresh = self._available_entries(clear_expired=True, refresh=True) if not available: - return None + return None, pending_refresh below_cap = [ entry for entry in available @@ -1869,7 +2111,7 @@ def acquire_lease(self, credential_id: Optional[str] = None) -> Optional[str]: ) self._active_leases[chosen.id] = self._active_leases.get(chosen.id, 0) + 1 self._current_id = chosen.id - return chosen.id + return chosen.id, pending_refresh def release_lease(self, credential_id: str) -> None: """Release a previously acquired credential lease.""" @@ -1885,9 +2127,11 @@ def try_refresh_current(self) -> Optional[PooledCredential]: return self._try_refresh_current_unlocked() def try_refresh_matching( - self, api_key_hint: Optional[str] = None + self, + api_key_hint: Optional[str] = None, + credential_id: Optional[str] = None, ) -> Optional[PooledCredential]: - """Force-refresh the entry that supplied ``api_key_hint``. + """Force-refresh the entry that supplied the failed request. Direct provider integrations may reload the pool after a request has already failed, so they cannot rely on ``current_id`` identifying the @@ -1897,17 +2141,29 @@ def try_refresh_matching( """ with self._lock: entry = None - if api_key_hint: + if credential_id: entry = next( ( candidate for candidate in self._entries - if candidate.runtime_api_key == api_key_hint + if candidate.id == credential_id ), None, ) - else: - entry = self._current_unlocked() or self._select_unlocked(refresh=False) + if entry is None: + if api_key_hint: + entry = next( + ( + candidate + for candidate in self._entries + if candidate.runtime_api_key == api_key_hint + ), + None, + ) + else: + entry = self._current_unlocked() or self._select_unlocked( + refresh=False + )[0] if entry is None: return None self._current_id = entry.id @@ -2235,31 +2491,75 @@ def _env_val(key: str) -> str: # env vars (COPILOT_GITHUB_TOKEN / GH_TOKEN). They don't live in # the auth store or credential pool, so we resolve them here. try: - from hermes_cli.copilot_auth import resolve_copilot_token, get_copilot_api_token + from hermes_cli.copilot_auth import ( + COPILOT_ENV_VARS, + resolve_copilot_token, + get_copilot_api_token, + ) + # All-sources suppression gate BEFORE any work — including the + # `gh auth token` subprocess spawn. resolve_copilot_token() + # shells out (~30ms), and the exchange retries 3x with backoff + # (~35s worst case); a user who suppressed every copilot source + # (hermes auth remove copilot gh_cli) must not pay either on + # every pool load (model picker open, /model, agent startup). + # Enumerating the full source space here matches what + # credential_sources._remove_copilot_gh suppresses, so an + # all-suppressed check is stable. + copilot_sources = ["gh_cli"] + [f"env:{v}" for v in COPILOT_ENV_VARS] + if all(_is_suppressed(provider, s) for s in copilot_sources): + return changed, active_sources token, source = resolve_copilot_token() if token: + # ``resolve_copilot_token`` returns exactly "gh auth token" + # for the CLI path; env-sourced tokens return the var name. + # Match exactly — a substring test classifies GH_TOKEN and + # GITHUB_TOKEN as gh_cli, silently bypassing a user's + # per-env-var suppression. + source_name = "gh_cli" if source == "gh auth token" else f"env:{source}" + # Per-source suppression gate (a user may suppress only the + # gh CLI path and keep an env var, or vice versa) BEFORE the + # network exchange. The exchange retries 3x with 10s + # timeouts and 4.5s total backoff (~35s worst case), so a + # source the user already suppressed + # must not burn that dead time just to have the entry + # discarded afterwards. Same early-gate pattern every other + # singleton branch uses. + if _is_suppressed(provider, source_name): + return changed, active_sources api_token, enterprise_base_url = get_copilot_api_token(token) - source_name = "gh_cli" if "gh" in source.lower() else f"env:{source}" - if not _is_suppressed(provider, source_name): - active_sources.add(source_name) - pconfig = PROVIDER_REGISTRY.get(provider) - # Use enterprise base URL from token exchange if available, - # otherwise fall back to the provider's default. - effective_base_url = enterprise_base_url or ( - pconfig.inference_base_url if pconfig else "" - ) - changed |= _upsert_entry( - entries, - provider, - source_name, - { - "source": source_name, - "auth_type": AUTH_TYPE_API_KEY, - "access_token": api_token, - "base_url": effective_base_url, - "label": source, - }, + # Observability: get_copilot_api_token falls back to returning + # the RAW token when the exchange fails. A raw ~40-char token + # sent to the Copilot API is routed to the fallback + # "copilot-language-server" integrator, whose allowlist omits + # enterprise-only models (claude-opus-4.8) → HTTP 400 on every + # turn. exchange_copilot_token now retries + reuses a persisted + # JWT, so this should be rare; surface it at WARNING so a + # recurrence is visible in logs instead of failing silently. + if api_token == token and not enterprise_base_url: + logger.warning( + "Copilot token exchange degraded to RAW token (exchange " + "unavailable); enterprise-only models may 400 with " + "model_not_available_for_integrator until exchange recovers." ) + active_sources.add(source_name) + pconfig = PROVIDER_REGISTRY.get(provider) + # Use enterprise base URL from token exchange if available, + # otherwise fall back to the provider's default. + effective_base_url = enterprise_base_url or ( + pconfig.inference_base_url if pconfig else "" + ) + changed |= _upsert_entry( + entries, + provider, + source_name, + { + "source": source_name, + "auth_type": AUTH_TYPE_API_KEY, + "access_token": api_token, + "base_url": effective_base_url, + "label": source, + }, + ) except Exception as exc: logger.debug("Copilot token seed failed: %s", exc) @@ -2409,6 +2709,20 @@ def _seed_from_env(provider: str, entries: List[PooledCredential]) -> Tuple[bool changed = False active_sources: Set[str] = set() + # Copilot has its own dedicated seeding branch (see `_seed_credentials` + # for provider == "copilot") which exchanges the raw ghu_ OAuth token + # for the ~437-char api token via `get_copilot_api_token`. If we let + # the generic env-var loop below run for copilot, it re-reads + # COPILOT_GITHUB_TOKEN from .env and shoves the RAW 40-char token in + # as `access_token`, overwriting the correctly-exchanged token. That + # bypasses the Copilot token exchange entirely and causes 400s with + # "not available for integrator copilot-language-server" (the server's + # fallback integrator when it receives a raw OAuth token instead of + # an api token). Skip the generic loop here — the copilot-specific + # branch is authoritative. + if provider == "copilot": + return False, active_sources + # Prefer ~/.hermes/.env over os.environ — the user's config file is the # authoritative source for Hermes credentials. Stale env vars from parent # processes (Codex CLI, test scripts, etc.) should not override deliberate diff --git a/agent/credential_sources.py b/agent/credential_sources.py index 18f0823ba842..32cd5e01a80d 100644 --- a/agent/credential_sources.py +++ b/agent/credential_sources.py @@ -164,7 +164,7 @@ def _remove_env_source(provider: str, removed) -> RemovalResult: if env_path.exists(): env_in_dotenv = any( line.strip().startswith(f"{env_var}=") - for line in env_path.read_text(errors="replace").splitlines() + for line in env_path.read_text(errors="replace", encoding="utf-8").splitlines() ) except OSError: pass diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 929bc34d3262..b47c3f274edf 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -170,6 +170,27 @@ def used_fraction(self) -> Optional[float]: ) CREDITS_USAGE_KEY = "credits.usage" # single key for the escalating usage notice +# Minimum subscription balance that counts as "grant not yet spent" for the +# grant_spent crossing gate (see evaluate_credits_notices). 1¢: portal-seeded +# states derive micros from float dollars and can carry sub-cent residue where +# the inference headers report exactly 0 — without this floor such a seed +# opens the gate and the first header re-creates the at-open nag. +GRANT_UNSPENT_MIN_MICROS = 10_000 + + +def new_credits_latch() -> dict: + """Fresh notice latch in the shape :func:`evaluate_credits_notices` expects. + + The policy owns this schema — every producer (agent build, lazy re-init, + tests) must build the latch through here so a new gate key lands everywhere + at once instead of drifting across hand-rolled literals.""" + return { + "active": set(), + "seen_below_90": False, + "usage_band": None, + "seen_grant_unspent": False, + } + # ── AgentNotice (out-of-band notice payload; driver-agnostic) ──────────────── @@ -250,7 +271,8 @@ def evaluate_credits_notices( ) -> tuple[list[AgentNotice], list[str]]: """Reconcile credits notices against the latch. Mutates ``latch`` IN PLACE. - latch = {"active": set[str], "seen_below_90": bool, "usage_band": Optional[int]}. + latch = {"active": set[str], "seen_below_90": bool, "usage_band": Optional[int], + "seen_grant_unspent": bool}. ``model_is_free``: True when the session's active model is a Nous free-tier model (see :func:`is_free_tier_model`). Suppresses the ``credits.depleted`` @@ -277,6 +299,18 @@ def evaluate_credits_notices( if uf is not None and uf < _lowest_band: latch["seen_below_90"] = True # gate opened: usage-band notices may now fire + # Grant-spent crossing gate: grant_spent may fire only after this session + # has OBSERVED the grant meaningfully unspent (≥1¢ left — see + # GRANT_UNSPENT_MIN_MICROS). Opening at grant-spent is a steady STATE, not + # an event — /usage carries it; only a live in-session crossing announces. + # Unlike seen_below_90, seeds must NOT prime this gate. + if ( + uf is not None + and uf < 1.0 + and state.subscription_micros >= GRANT_UNSPENT_MIN_MICROS + ): + latch["seen_grant_unspent"] = True + active = latch["active"] # ── Conditions ─────────────────────────────────────────────────────────── @@ -341,7 +375,17 @@ def evaluate_credits_notices( latch["usage_band"] = target_band # ── grant_spent ────────────────────────────────────────────────────────── - if grant_cond and "credits.grant_spent" not in active: + # The crossing gate guards only the SHOW and is CONSUMED by it — one + # announcement per crossing. A header flicker (uf → None → back to 1.0) + # clears the sticky line via grant_cond but cannot re-announce; only a + # renewal that re-opens the gate (a fresh ≥1¢ observation) arms the next + # announcement. .get(): default closed for any hand-built latch missing + # the key, so a first observation can never fire this notice. + if ( + grant_cond + and "credits.grant_spent" not in active + and latch.get("seen_grant_unspent", False) + ): to_show.append( AgentNotice( text=f"• Grant spent · ${state.purchased_usd} top-up left", @@ -352,6 +396,7 @@ def evaluate_credits_notices( ) ) active.add("credits.grant_spent") + latch["seen_grant_unspent"] = False elif "credits.grant_spent" in active and not grant_cond: to_clear.append("credits.grant_spent") active.discard("credits.grant_spent") @@ -627,7 +672,8 @@ def _req_int(key: str) -> Any: subscription_limit_micros=20_000_000, subscription_limit_usd="20.00", denominator_kind="subscription_cap", paid_access=True, ), - "grant_exhausted": dict( # used_fraction == 1.0 + purchased>0 → credits.grant_spent + "grant_exhausted": dict( # uf == 1.0 + purchased>0 → SILENT at open (crossing-gated); + # flip healthy → grant_exhausted via the fixture-file path to see credits.grant_spent remaining_micros=12_340_000, remaining_usd="12.34", subscription_micros=0, subscription_usd="0.00", subscription_limit_micros=20_000_000, subscription_limit_usd="20.00", @@ -741,6 +787,9 @@ def _hydrate_seed_state(agent, state) -> None: agent._credits_session_start_micros = state.remaining_micros _latch = getattr(agent, "_credits_latch", None) if isinstance(_latch, dict) and state.used_fraction is not None: + # Prime ONLY seen_below_90 (open-high band warnings are wanted at open). + # Never prime seen_grant_unspent here: a seed observing grant-spent is a + # steady state, and priming it would revive the every-session nag. _latch["seen_below_90"] = True emit = getattr(agent, "_emit_credits_notices", None) if callable(emit): diff --git a/agent/curator.py b/agent/curator.py index dc908fc35939..ab0adddae118 100644 --- a/agent/curator.py +++ b/agent/curator.py @@ -138,8 +138,8 @@ def is_paused() -> bool: def _load_config() -> Dict[str, Any]: """Read curator.* config from ~/.hermes/config.yaml. Tolerates missing file.""" try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() except Exception as e: logger.debug("Failed to load config for curator: %s", e) return {} @@ -325,7 +325,7 @@ def apply_automatic_transitions(now: Optional[datetime] = None) -> Dict[str, int counts = {"marked_stale": 0, "archived": 0, "reactivated": 0, "checked": 0, "seeded": 0} - for row in _u.agent_created_report(): + for row in _u.curated_report(): counts["checked"] += 1 name = row["name"] if row.get("pinned"): @@ -902,7 +902,6 @@ def _reconcile_classification( Every removed skill is placed in exactly one bucket. """ heur_cons = {e["name"]: e for e in heuristic.get("consolidated", [])} - heur_pruned = {e["name"] for e in heuristic.get("pruned", [])} model_cons = {e["from"]: e for e in model_block.get("consolidations", [])} model_pruned = {e["name"]: e for e in model_block.get("prunings", [])} @@ -1472,15 +1471,16 @@ def _render_report_markdown(p: Dict[str, Any]) -> str: # --------------------------------------------------------------------------- def _render_candidate_list() -> str: - """Human/agent-readable list of agent-created skills with usage stats.""" - rows = skill_usage.agent_created_report() + """Human/agent-readable list of curator-managed skills with usage stats.""" + rows = skill_usage.curated_report() if not rows: - return "No agent-created skills to review." + return "No curator-managed skills to review." cron_referenced = _cron_referenced_skills() - lines = [f"Agent-created skills ({len(rows)}):\n"] + lines = [f"Curator-managed skills ({len(rows)}):\n"] for r in rows: lines.append( f"- {r['name']} " + f"provenance={r.get('provenance', 'agent')} " f"state={r['state']} " f"pinned={'yes' if r.get('pinned') else 'no'} " f"cron={'yes' if r['name'] in cron_referenced else 'no'} " @@ -1533,7 +1533,7 @@ def run_curator_review( if dry_run: # Count candidates without mutating state. try: - report = skill_usage.agent_created_report() + report = skill_usage.curated_report() counts = { "checked": len(report), "marked_stale": 0, @@ -1586,7 +1586,7 @@ def _llm_pass(): nonlocal auto_summary # Snapshot skill state BEFORE the LLM pass so the report can diff. try: - before_report = skill_usage.agent_created_report() + before_report = skill_usage.curated_report() except Exception: before_report = [] before_names = {r.get("name") for r in before_report if isinstance(r, dict)} @@ -1612,7 +1612,7 @@ def _llm_pass(): state2["last_run_duration_seconds"] = elapsed state2["last_run_summary"] = final_summary try: - after_report = skill_usage.agent_created_report() + after_report = skill_usage.curated_report() except Exception: after_report = [] try: @@ -1699,7 +1699,7 @@ def _llm_pass(): try: rename_lines = _build_rename_summary( before_names=before_names, - after_report=skill_usage.agent_created_report(), + after_report=skill_usage.curated_report(), tool_calls=llm_meta.get("tool_calls", []) or [], model_final=llm_meta.get("final", "") or "", ) @@ -1717,7 +1717,7 @@ def _llm_pass(): # reporting bug never breaks the curator itself. Report path is # recorded in state so `hermes curator status` can point at it. try: - after_report = skill_usage.agent_created_report() + after_report = skill_usage.curated_report() except Exception: after_report = [] try: @@ -1875,9 +1875,9 @@ def _run_llm_review(prompt: str) -> Dict[str, Any]: _acp_args = None _model_name = "" try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly from hermes_cli.runtime_provider import resolve_runtime_provider - _cfg = load_config() + _cfg = load_config_readonly() _binding = _resolve_review_runtime(_cfg) _provider, _model_name = _binding.provider, _binding.model _rp = resolve_runtime_provider( @@ -1923,6 +1923,7 @@ def _run_llm_review(prompt: str) -> Dict[str, Any]: credential_pool=_credential_pool, request_overrides=_request_overrides, **_agent_kwargs, + enabled_toolsets=["skills", "terminal"], # Umbrella-building over a large skill collection is worth a # high iteration ceiling — the pass typically takes 50-100 # API calls against hundreds of candidate skills. The diff --git a/agent/curator_backup.py b/agent/curator_backup.py index 5b95f9e3e707..8a65825464e4 100644 --- a/agent/curator_backup.py +++ b/agent/curator_backup.py @@ -147,8 +147,8 @@ def _utc_id(now: Optional[datetime] = None) -> str: def _load_config() -> Dict[str, Any]: try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() except Exception as e: logger.debug("Failed to load config for curator backup: %s", e) return {} @@ -541,6 +541,33 @@ def _restore_cron_skill_links(snapshot_dir: Path) -> Dict[str, Any]: +def _unstage(moved: List[Tuple[Path, Path]]) -> List[str]: + """Move staged entries back to their original paths. + + ``shutil.move`` moves *into* an existing destination directory rather than + replacing it, so a partially-completed extract leaves debris that would + otherwise bury the user's real skill one level deeper + (``skills/foo/foo/``) while the tree still looks populated. Clear whatever + the failed extract created at each original path first. The staged copy is + authoritative, and the pre-rollback safety snapshot is the undo handle for + the extract's own output. + + Returns the names that could not be restored, so the caller can report an + incomplete recovery instead of claiming the state was restored. + """ + failed: List[str] = [] + for orig, dest in moved: + try: + if orig.is_dir() and not orig.is_symlink(): + shutil.rmtree(orig) + elif orig.exists() or orig.is_symlink(): + orig.unlink() + shutil.move(str(dest), str(orig)) + except OSError: + failed.append(orig.name) + return failed + + def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path]]: """Restore ``~/.hermes/skills/`` from a snapshot. @@ -609,11 +636,7 @@ def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path] moved.append((entry, dest)) except OSError as e: # Best-effort rollback of the move - for orig, dest in moved: - try: - shutil.move(str(dest), str(orig)) - except OSError: - pass + _unstage(moved) try: shutil.rmtree(staged, ignore_errors=True) except OSError: @@ -638,12 +661,30 @@ def rollback(backup_id: Optional[str] = None) -> Tuple[bool, str, Optional[Path] # Python < 3.12 — no filter kwarg tf.extractall(str(skills)) except (OSError, tarfile.TarError) as e: - # Best-effort recover: move staged contents back - for orig, dest in moved: + # Best-effort recover. A partial extract can leave entries the + # original tree never had, so drop those first, otherwise the + # "restored" tree is the user's skills plus a slice of the snapshot. + staged_names = {orig.name for orig, _ in moved} + for entry in list(skills.iterdir()): + if entry.name in _EXCLUDE_TOP_LEVEL or entry.name in staged_names: + continue try: - shutil.move(str(dest), str(orig)) + if entry.is_dir() and not entry.is_symlink(): + shutil.rmtree(entry) + else: + entry.unlink() except OSError: pass + unrestored = _unstage(moved) + if unrestored: + # Do not claim a clean restore we did not achieve, and keep the + # staging dir so the entries can be recovered by hand. + return ( + False, + f"snapshot extract failed: {e} - could not restore " + f"{', '.join(sorted(unrestored))}; staged copies kept at {staged}", + None, + ) try: shutil.rmtree(staged, ignore_errors=True) except OSError: diff --git a/agent/delegation_context.py b/agent/delegation_context.py index b80bbbe00fbf..41fe7f56ac17 100644 --- a/agent/delegation_context.py +++ b/agent/delegation_context.py @@ -31,11 +31,21 @@ @contextmanager -def delegated_child_context() -> Iterator[None]: - """Mark the current execution context as a delegate_task child.""" +def delegated_child_context(session_id: str | None = None) -> Iterator[None]: + """Mark child execution and isolate its task-local session identity. + + Child construction calls ``set_current_session_id`` internally, so even a + context entered without an id must restore the parent's ContextVar. Child + execution passes its explicit id and receives it only for this scope. + """ token = _DELEGATED_CHILD_CONTEXT.set(True) try: - yield + # Import lazily: session_context calls is_delegated_child_context() when + # deciding whether the compatibility os.environ mirror is safe. + from gateway.session_context import scoped_current_session_id + + with scoped_current_session_id(session_id): + yield finally: _DELEGATED_CHILD_CONTEXT.reset(token) diff --git a/agent/display.py b/agent/display.py index 3da6e2f24ec8..d6bea7e54e87 100644 --- a/agent/display.py +++ b/agent/display.py @@ -14,6 +14,7 @@ from difflib import unified_diff from pathlib import Path from typing import Any +from urllib.parse import urlsplit from utils import safe_json_loads from agent.redact import redact_sensitive_text @@ -187,6 +188,15 @@ def _truncate_preview(text: str, max_len: int | None) -> str: return text +@dataclass(frozen=True) +class ToolPreview: + """A compact tool preview plus presentation facts lost to truncation.""" + + text: str + truncated: bool = False + url: str | None = None + + _SHELL_SILENT_HEADS = {"cd", "pushd", "popd", "export", "set", "unset", "source", ".", "true", "false", ":"} _SHELL_PIPE_TAIL_HEADS = {"head", "tail", "wc", "sort", "uniq"} @@ -556,6 +566,35 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) - return preview +def prepare_tool_preview( + tool_name: str, + args: dict | None, + *, + fallback: str, + max_len: int, +) -> ToolPreview: + """Build one canonical compact preview before platform formatting. + + The uncapped preview is rebuilt from the tool arguments when possible so + an upstream display cap cannot discard its link target. Platforms then + receive explicit truncation and URL metadata instead of inferring either + fact from the rendered text. + """ + full_text = build_tool_preview(tool_name, args, max_len=0) or fallback + text = _truncate_preview(full_text, max_len) + truncated = text != full_text + url = None + if truncated: + candidate = _display_url(full_text) + try: + parsed = urlsplit(candidate) + except ValueError: + parsed = None + if parsed and parsed.scheme.lower() in {"http", "https"} and parsed.netloc: + url = candidate + return ToolPreview(text=text, truncated=truncated, url=url) + + # ========================================================================= # Friendly tool labels (human-phrased verbs for built-in tools) # @@ -1506,5 +1545,3 @@ def get_cute_tool_message( # ========================================================================= # Honcho session line (one-liner with clickable OSC 8 hyperlink) # ========================================================================= - - diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 33c2f5458560..8ac0b6c87234 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -159,6 +159,14 @@ def is_auth(self) -> bool: "throttlingexception", "too many concurrent requests", "servicequotaexceededexception", + # Generic throttle prefix — Bedrock (and some proxies) surface throttling + # as "Throttling error: Too many tokens, please wait before trying + # again." Without this entry the message falls through to the + # context-overflow list (which contains "too many tokens") and the retry + # loop compresses a healthy session instead of backing off. Matched + # BEFORE _CONTEXT_OVERFLOW_PATTERNS in the message-only path, so the + # throttle wins. (port of anomalyco/opencode#37848's exclusion guard) + "throttling", ] # Patterns that indicate provider-side overload, NOT a per-credential rate @@ -212,6 +220,12 @@ def is_auth(self) -> bool: "request entity too large", "payload too large", "error code: 413", + # Anthropic's structured 413 error type. Normally arrives with an HTTP + # 413 status (handled by the status path), but aggregators/proxies can + # re-wrap it into a plain message with no status attribute — route it to + # the same compression recovery. (port of anomalyco/opencode#37848) + "request_too_large", + "request exceeds the maximum size", ] # Image-size patterns. Matched against 400 bodies (not 413) because most @@ -298,6 +312,10 @@ def is_auth(self) -> bool: "max input token", "input token", "exceeds the maximum number of input tokens", + # Together/Fireworks-style: "Input length 131393 exceeds the maximum + # allowed input length of 131040 tokens." No other pattern in this list + # matches that wording. (port of anomalyco/opencode#37848) + "maximum allowed input length", ] # Model not found patterns @@ -321,6 +339,30 @@ def is_auth(self) -> bool: "no endpoints found that support tool use", ] +# Malformed-message-array 400s. Deterministic request-shape rejections that +# describe the *transcript* being invalid, not a parameter. The canonical +# case: a stream dies mid-response and Hermes persists a content-less +# assistant stub; on the next turn the Anthropic message schema (and the +# litellm/Bedrock proxies in front of it) reject the whole request with +# "all messages must have non-empty content except for the optional final +# assistant message" / errorCode INVALID_REQUEST_BODY +# These are NOT context overflow — the input may be tiny — but a large +# session used to mis-route them into the compression loop via the generic +# "400 + large session" heuristic below, ending in "Cannot compress further" +# every retry (the input is unchanged, so compression cannot help). Match +# the message-shape signals explicitly and fail fast as a format_error so the +# loop stops looping. The empty-stub creation is the root cause (fixed in +# chat_completion_helpers); this pattern stops the misclassification symptom +# for transcripts that already contain a poisoned stub. +_INVALID_MESSAGE_BODY_PATTERNS = [ + "must have non-empty content", + "messages must have non-empty", + "invalid_request_body", + "text content blocks must be non-empty", + "content field is required", + "messages: at least one message is required", +] + # Request-validation patterns — the request is malformed and will fail # identically on every retry. Some OpenAI-compatible gateways (notably # codex.nekos.me) return these as 5xx instead of the standard 4xx, which @@ -794,6 +836,19 @@ def _result(reason: FailoverReason, **overrides) -> ClassifiedError: if classified is not None: return classified + # Local MoA streaming compatibility errors are adapter-shape bugs, not a + # provider outage. Falling back to another model would silently switch the + # user's selected MoA route to a single-model answer (#55933 follow-up). + if provider_lower == "moa" and ( + "'types.SimpleNamespace' object is not iterable" in str(error) + or "'types.SimpleNamespace' object has no attribute 'index'" in str(error) + ): + return _result( + FailoverReason.format_error, + retryable=False, + should_fallback=False, + ) + # Local MoA config drift is deterministic: a persisted session can retain # a preset name that was later renamed/deleted. Retrying the same lookup # cannot recover and makes a clear config error look like an API outage. @@ -1271,6 +1326,33 @@ def _classify_400( should_fallback=True, ) + # Malformed message array (empty-content assistant stub, etc.). Must be + # checked BEFORE context_overflow: the input can be tiny, so the generic + # "400 + large session" heuristic would otherwise mis-route it into the + # compression loop and thrash until "Cannot compress further" on every + # retry (the request is unchanged, so compression cannot fix it). This is + # a deterministic request-shape rejection — fail fast as a non-retryable + # format_error and fall back. Checked against the message text AND the + # structured error code, since proxies (litellm/Bedrock) surface the + # signal in errorCode=INVALID_REQUEST_BODY. + if ( + any(p in error_msg for p in _INVALID_MESSAGE_BODY_PATTERNS) + or error_code_lower == "invalid_request_body" + ): + logger.warning( + "Malformed message array 400 (invalid request body) classified as " + "format_error, NOT context overflow — failing fast + falling back " + "instead of entering the compression loop. This usually means an " + "empty-content assistant stub is in the transcript; num_messages=%s " + "approx_tokens=%s. error=%.200s", + num_messages, approx_tokens, error_msg, + ) + return result_fn( + FailoverReason.format_error, + retryable=False, + should_fallback=True, + ) + # Empty-provider-response advisories must not enter compression. They # often mention "max_tokens" as a possible cause and used to match the # bare overflow pattern, then thrash compress until "Cannot compress @@ -1331,6 +1413,18 @@ def _classify_400( # Responses API (and some providers) use flat body: {"message": "..."} if not err_body_msg: err_body_msg = str(body.get("message") or "").strip().lower() + # litellm / Bedrock proxies use a custom shape: {"errorMessage": "...", + # "errorCode": "...", "errorArgs": {"reason": "..."}}. Without these + # keys err_body_msg stays "" and a long, descriptive rejection is + # wrongly treated as a "generic" (bare) error below, which — on a + # large session — mis-routes into the compression loop. Recognize + # them so the is_generic heuristic sees the real message length. + if not err_body_msg: + err_body_msg = str(body.get("errorMessage") or "").strip().lower() + if not err_body_msg: + _args = body.get("errorArgs") + if isinstance(_args, dict): + err_body_msg = str(_args.get("reason") or "").strip().lower() is_generic = len(err_body_msg) < 30 or err_body_msg in {"error", ""} # Absolute token/message-count thresholds are only a proxy for smaller # context windows. Large-context sessions can have many messages while @@ -1629,7 +1723,7 @@ def _code_from_payload(payload) -> str: return nested_code # Top-level code - code = body.get("code") or body.get("error_code") or "" + code = body.get("code") or body.get("error_code") or body.get("errorCode") or "" if isinstance(code, (str, int)): text = str(code).strip() if text and text != "400": @@ -1649,6 +1743,16 @@ def _extract_message(error: Exception, body: dict) -> str: msg = body.get("message", "") if isinstance(msg, str) and msg.strip(): return msg.strip()[:500] + # litellm / Bedrock proxy shape: {"errorMessage": "...", + # "errorArgs": {"reason": "..."}}. + msg = body.get("errorMessage", "") + if isinstance(msg, str) and msg.strip(): + return msg.strip()[:500] + args = body.get("errorArgs") + if isinstance(args, dict): + reason = args.get("reason", "") + if isinstance(reason, str) and reason.strip(): + return reason.strip()[:500] # Fallback to str(error) return str(error)[:500] diff --git a/agent/gemini_native_adapter.py b/agent/gemini_native_adapter.py index b4f6e6386e7d..cde63f15fc17 100644 --- a/agent/gemini_native_adapter.py +++ b/agent/gemini_native_adapter.py @@ -73,7 +73,7 @@ def probe_gemini_tier( api_key: str, base_url: str = DEFAULT_GEMINI_BASE_URL, *, - model: str = "gemini-2.5-flash", + model: str = "gemini-3.6-flash", timeout: float = 10.0, ) -> str: """Probe a Google AI Studio API key and return its tier. @@ -154,8 +154,8 @@ def is_free_tier_quota_error(error_message: str) -> bool: _FREE_TIER_GUIDANCE = ( - "\n\nYour Google API key is on the free tier (<= 250 requests/day for " - "gemini-2.5-flash). Hermes typically makes 3-10 API calls per user turn, " + "\n\nYour Google API key is on the free tier (a few hundred requests/day " + "for Gemini Flash models). Hermes typically makes 3-10 API calls per user turn, " "so the free tier is exhausted in a handful of messages and cannot sustain " "an agent session. Enable billing on your Google Cloud project and " "regenerate the key in a billing-enabled project: " @@ -163,6 +163,42 @@ def is_free_tier_quota_error(error_message: str) -> bool: ) +def is_standard_key_auth_error( + status: int, error_message: str, reason: str = "" +) -> bool: + """Return True when a Gemini 401 indicates Google rejected the key TYPE. + + Google began rejecting unrestricted legacy "Standard" Google Cloud API + keys on the Gemini API on June 19, 2026, and ALL Standard keys stop + working in September 2026. The rejection surfaces as a misleading 401 + telling the user to supply an OAuth 2 access token ("Request had invalid + authentication credentials. Expected OAuth 2 access token, login cookie + or other valid authentication credential."), optionally carrying + ``google.rpc.ErrorInfo`` reason ``ACCESS_TOKEN_TYPE_UNSUPPORTED``. + + Scoped narrowly so a plain bad key (reason ``API_KEY_INVALID``, + "API key not valid") keeps its existing message. + """ + if status != 401: + return False + if reason == "ACCESS_TOKEN_TYPE_UNSUPPORTED": + return True + return "expected oauth 2 access token" in (error_message or "").lower() + + +_STANDARD_KEY_GUIDANCE = ( + "\n\nGoogle Gemini rejected this API key's type — you do NOT need OAuth. " + "Google began rejecting legacy 'Standard' Google Cloud keys for the " + "Gemini API on June 19, 2026, and all Standard keys stop working in " + "September 2026. Open https://aistudio.google.com/api-keys, check the " + "key's type and status, and create a replacement Gemini API key (or, as " + "a temporary bridge, restrict the Standard key to " + "generativelanguage.googleapis.com). Then update GEMINI_API_KEY / " + "GOOGLE_API_KEY in ~/.hermes/.env and restart your session. " + "Details: https://ai.google.dev/gemini-api/docs/api-key" +) + + class GeminiAPIError(Exception): """Error shape compatible with Hermes retry/error classification.""" @@ -285,9 +321,13 @@ def _translate_tool_result_to_gemini( ) -> Dict[str, Any]: tool_name_by_call_id = tool_name_by_call_id or {} tool_call_id = str(message.get("tool_call_id") or "") + # A tool result can carry the unwrapped internal tool name (for example, + # an MCP tool invoked through the `tool_call` bridge). Gemini requires + # functionResponse.name to echo the matching functionCall.name, so the + # call-id mapping must take precedence over the internal result name. name = str( - message.get("name") - or tool_name_by_call_id.get(tool_call_id) + tool_name_by_call_id.get(tool_call_id) + or message.get("name") or tool_call_id or "tool" ) @@ -824,6 +864,12 @@ def gemini_http_error( if status == 429 and is_free_tier_quota_error(err_message or body_text): message = message + _FREE_TIER_GUIDANCE + # Legacy "Standard" Google Cloud key rejection (June 19, 2026 onward) -> + # Google's raw 401 misleadingly tells the user to use OAuth. Append the + # actual fix (mint a new Gemini API key in AI Studio). + if is_standard_key_auth_error(status, err_message or body_text, reason): + message = message + _STANDARD_KEY_GUIDANCE + return GeminiAPIError( message, code=code, @@ -934,7 +980,7 @@ def _advance_stream_iterator(iterator: Iterator[_GeminiStreamChunk]) -> tuple[bo def _create_chat_completion( self, *, - model: str = "gemini-2.5-flash", + model: str = "gemini-3.6-flash", messages: Optional[List[Dict[str, Any]]] = None, stream: bool = False, tools: Any = None, diff --git a/agent/gemini_schema.py b/agent/gemini_schema.py index b0985422fbb1..665fd79a37e4 100644 --- a/agent/gemini_schema.py +++ b/agent/gemini_schema.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from typing import Any, Dict # Gemini's ``FunctionDeclaration.parameters`` field accepts the ``Schema`` @@ -76,15 +77,31 @@ def sanitize_gemini_schema(schema: Any) -> Dict[str, Any]: # Gemini's Schema validator requires every ``enum`` entry to be a string, # even when the parent ``type`` is ``integer`` / ``number`` / ``boolean``. - # OpenAI / OpenRouter / Anthropic accept typed enums (e.g. Discord's - # ``auto_archive_duration: {type: integer, enum: [60, 1440, 4320, 10080]}``), - # so we only drop the ``enum`` when it would collide with Gemini's rule. - # Keeping ``type: integer`` plus the human-readable description gives the - # model enough guidance; the tool handler still validates the value. + # Preserve those constraints by stringifying scalar values while keeping + # the declared type intact; Gemini uses the strings as schema metadata and + # still emits typed tool arguments at runtime. enum_val = cleaned.get("enum") type_val = cleaned.get("type") if isinstance(enum_val, list) and type_val in {"integer", "number", "boolean"}: - if any(not isinstance(item, str) for item in enum_val): + stringified = [] + for item in enum_val: + if isinstance(item, str): + value = item + elif isinstance(item, bool): + value = "true" if item else "false" + elif ( + isinstance(item, (int, float)) + and not isinstance(item, bool) + and math.isfinite(item) + ): + value = str(item) + else: + continue + if value not in stringified: + stringified.append(value) + if stringified: + cleaned["enum"] = stringified + else: cleaned.pop("enum", None) # Gemini validates ``required`` strictly against the same node's diff --git a/agent/google_antigravity_cli_adapter.py b/agent/google_antigravity_cli_adapter.py index db8adf4ca78e..d83535411b7c 100644 --- a/agent/google_antigravity_cli_adapter.py +++ b/agent/google_antigravity_cli_adapter.py @@ -98,7 +98,7 @@ def create(self, **kwargs: Any) -> Any: prompt = build_prompt_from_messages(messages) if not prompt: raise AntigravityCLIError("Antigravity CLI request had no text prompt") - content = self._client.complete(prompt) + content = self._client.complete(prompt, timeout=kwargs.get("timeout")) return _completion_response(content, model) @@ -119,6 +119,7 @@ def __init__( api_key: str = "google-antigravity-cli", model: str = DEFAULT_ANTIGRAVITY_MODEL, print_timeout: str = DEFAULT_PRINT_TIMEOUT, + subprocess_timeout: float = 330.0, ) -> None: self.command = command self.args = list(args or []) @@ -126,10 +127,17 @@ def __init__( self.api_key = api_key self.model = model self.print_timeout = print_timeout + self.subprocess_timeout = subprocess_timeout self.chat = _AntigravityChat(self) - def complete(self, prompt: str) -> str: + def complete(self, prompt: str, *, timeout: Any = None) -> str: cmd = [self.command, *self.args, "-p", prompt, "--print-timeout", self.print_timeout] + effective_timeout = self.subprocess_timeout + if timeout is not None: + try: + effective_timeout = min(effective_timeout, float(timeout)) + except (TypeError, ValueError): + pass try: proc = subprocess.run( cmd, @@ -137,11 +145,16 @@ def complete(self, prompt: str) -> str: text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=effective_timeout, ) except FileNotFoundError as exc: raise AntigravityCLIError( f"Antigravity CLI command not found: {self.command!r}" ) from exc + except subprocess.TimeoutExpired as exc: + raise AntigravityCLIError( + f"Antigravity CLI timed out after {effective_timeout:g}s" + ) from exc except Exception as exc: raise AntigravityCLIError( f"Antigravity CLI invocation failed: {exc.__class__.__name__}: {exc}" diff --git a/agent/i18n.py b/agent/i18n.py index 8ca1e2123f7d..7c0dcf5c87e7 100644 --- a/agent/i18n.py +++ b/agent/i18n.py @@ -25,7 +25,8 @@ 3. ``display.language`` from config.yaml 4. ``"en"`` (baseline) -Supported languages: en, zh, ja, de, es, fr, tr, uk. Unknown values fall back to en. +Supported languages: en, zh, zh-hant, ja, de, es, fr, tr, uk, af, ko, it, ga, +pt, ru, hu, ar. Unknown values fall back to en. """ from __future__ import annotations @@ -37,13 +38,11 @@ from pathlib import Path from typing import Any -from hermes_constants import find_packaged_data_dir - logger = logging.getLogger(__name__) SUPPORTED_LANGUAGES: tuple[str, ...] = ( "en", "zh", "zh-hant", "ja", "de", "es", "fr", "tr", "uk", - "af", "ko", "it", "ga", "pt", "ru", "hu", + "af", "ko", "it", "ga", "pt", "ru", "hu", "ar", ) DEFAULT_LANGUAGE = "en" @@ -80,6 +79,9 @@ "russian": "ru", "русский": "ru", "ru-ru": "ru", # Hungarian "hungarian": "hu", "magyar": "hu", "hu-hu": "hu", + # Arabic — bare "arabic"/endonym plus the common regional BCP-47 tags. + "arabic": "ar", "العربية": "ar", + "ar-sa": "ar", "ar-eg": "ar", "ar-ae": "ar", "ar-ma": "ar", "ar-dz": "ar", } _catalog_cache: dict[str, dict[str, str]] = {} @@ -95,7 +97,6 @@ def _locales_dir() -> Path: sealed-packaging system) to point at the installed catalog directory. 2. ``/locales`` -- source checkouts and editable installs, where the working tree sits next to ``agent/``. - 3. The interpreter data scheme -- regular wheel installs. Falling through to the source-style path (even when missing) keeps ``_load_catalog`` error messages informative -- it logs the path it @@ -114,11 +115,6 @@ def _locales_dir() -> Path: # agent/i18n.py -> agent/ -> repo root (source checkout, editable install) source_dir = Path(__file__).resolve().parent.parent / "locales" - if source_dir.is_dir(): - return source_dir - packaged_dir = find_packaged_data_dir("locales") - if packaged_dir is not None: - return packaged_dir return source_dir @@ -201,8 +197,8 @@ def _config_language_cached() -> str | None: (e.g. after the setup wizard). """ try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() lang = (cfg.get("display") or {}).get("language") if lang: return _normalize_lang(lang) diff --git a/agent/image_gen_registry.py b/agent/image_gen_registry.py index 5d14a6f1ece4..47538c8cf25e 100644 --- a/agent/image_gen_registry.py +++ b/agent/image_gen_registry.py @@ -91,9 +91,9 @@ def get_active_provider() -> Optional[ImageGenProvider]: """ configured: Optional[str] = None try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - cfg = load_config() + cfg = load_config_readonly() section = cfg.get("image_gen") if isinstance(cfg, dict) else None if isinstance(section, dict): raw = section.get("provider") diff --git a/agent/insights.py b/agent/insights.py index 086150c279ef..34e78a6ff9d6 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -99,6 +99,31 @@ def __init__(self, db): """ self.db = db self._conn = db._conn + # INDEXED BY is a hard dependency (SQLite errors on a missing index). + # A read-only open of a state.db written by an older version skips + # schema init and lacks the partial index — probe once and fall back + # to the unpinned variants (identical rows, optimizer-chosen plan). + try: + self._has_assistant_calls_index = bool( + self._conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?", + (self._MESSAGES_ASSISTANT_CALLS_INDEX,), + ).fetchone() + ) + except sqlite3.Error: + self._has_assistant_calls_index = False + if not self._has_assistant_calls_index: + _strip = f" INDEXED BY {self._MESSAGES_ASSISTANT_CALLS_INDEX}" + # Loop over every pinned statement so adding a new one can't + # forget its strip line (which would be a hard `no such index` + # crash on read-only DBs — the exact bug this fallback prevents). + for _attr in ( + "_GET_TOOL_CALLS_WITH_SOURCE", + "_GET_TOOL_CALLS_ALL", + "_GET_SKILL_CALLS_WITH_SOURCE", + "_GET_SKILL_CALLS_ALL", + ): + setattr(self, _attr, getattr(self, _attr).replace(_strip, "")) def generate(self, days: int = 30, source: str = None) -> Dict[str, Any]: """ @@ -113,6 +138,13 @@ def generate(self, days: int = 30, source: str = None) -> Dict[str, Any]: """ cutoff = time.time() - (days * 86400) + # Token/cost totals may still sit on the SessionDB's async + # accounting queue; drain so the report reflects exact counters. + # (self.db may be a raw sqlite3 connection in tests — guard.) + flush = getattr(self.db, "flush_token_counts", None) + if callable(flush): + flush() + # Gather raw data sessions = self._get_sessions(cutoff, source) tool_usage = self._get_tool_usage(cutoff, source) @@ -164,6 +196,21 @@ def generate(self, days: int = 30, source: str = None) -> Dict[str, Any]: "top_sessions": top_sessions, } + def get_usage_breakdown(self, days: int = 30, source: str = None) -> Dict[str, Any]: + """Return the analytics-usage payload without running a full generate(). + + Uses the instr()-prefiltered _get_skill_usage query so only messages + that reference skill_view or skill_manage are loaded from SQLite, while + still preserving the per-tool breakdown used by the dashboard route. + """ + cutoff = time.time() - (days * 86400) + tool_usage = self._get_tool_usage(cutoff, source) + skill_usage = self._get_skill_usage(cutoff, source) + return { + "tools": self._compute_tool_breakdown(tool_usage), + "skills": self._compute_skill_breakdown(skill_usage), + } + # ========================================================================= # Data gathering (SQL queries) # ========================================================================= @@ -188,6 +235,53 @@ def generate(self, days: int = 30, source: str = None) -> Dict[str, Any]: " ORDER BY started_at DESC" ) + # Assistant ``tool_calls`` scan for tool/skill usage. ``INDEXED BY`` pins + # the partial index ``idx_messages_assistant_calls_by_session`` so the plan + # is deterministic on a freshly initialized state.db (before ANALYZE has + # run) for BOTH the unfiltered and source-filtered branches — without the + # hint the optimizer falls back to ``idx_messages_session_active`` for the + # source-filtered probe and scans each session's non-tool-call rows. + # + # The pin is a HARD dependency: SQLite raises ``no such index`` when the + # named index is absent. That happens in practice — the web dashboard's + # usage analytics open the DB ``read_only=True`` (skipping + # ``_init_schema``), so a state.db created by an older writer has no + # partial index yet. ``__init__`` probes for the index once and falls + # back to the unpinned (still-correct, just optimizer-chosen) variants. + _MESSAGES_ASSISTANT_CALLS_INDEX = "idx_messages_assistant_calls_by_session" + _GET_TOOL_CALLS_WITH_SOURCE = ( + "SELECT m.tool_calls" + f" FROM messages m INDEXED BY {_MESSAGES_ASSISTANT_CALLS_INDEX}" + " JOIN sessions s ON s.id = m.session_id" + " WHERE s.started_at >= ? AND s.source = ?" + " AND m.role = 'assistant' AND m.tool_calls IS NOT NULL" + ) + _GET_TOOL_CALLS_ALL = ( + "SELECT m.tool_calls" + f" FROM messages m INDEXED BY {_MESSAGES_ASSISTANT_CALLS_INDEX}" + " JOIN sessions s ON s.id = m.session_id" + " WHERE s.started_at >= ?" + " AND m.role = 'assistant' AND m.tool_calls IS NOT NULL" + ) + _GET_SKILL_CALLS_WITH_SOURCE = ( + "SELECT m.tool_calls, m.timestamp" + f" FROM messages m INDEXED BY {_MESSAGES_ASSISTANT_CALLS_INDEX}" + " JOIN sessions s ON s.id = m.session_id" + " WHERE s.started_at >= ? AND s.source = ?" + " AND m.role = 'assistant' AND m.tool_calls IS NOT NULL" + " AND (instr(m.tool_calls, 'skill_view') > 0" + " OR instr(m.tool_calls, 'skill_manage') > 0)" + ) + _GET_SKILL_CALLS_ALL = ( + "SELECT m.tool_calls, m.timestamp" + f" FROM messages m INDEXED BY {_MESSAGES_ASSISTANT_CALLS_INDEX}" + " JOIN sessions s ON s.id = m.session_id" + " WHERE s.started_at >= ?" + " AND m.role = 'assistant' AND m.tool_calls IS NOT NULL" + " AND (instr(m.tool_calls, 'skill_view') > 0" + " OR instr(m.tool_calls, 'skill_manage') > 0)" + ) + def _get_sessions(self, cutoff: float, source: str = None) -> List[Dict]: """Fetch sessions within the time window.""" if source: @@ -236,22 +330,10 @@ def _get_tool_usage(self, cutoff: float, source: str = None) -> List[Dict]: # (covers CLI sessions where tool_name is NULL on tool responses) if source: cursor2 = self._conn.execute( - """SELECT m.tool_calls - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE s.started_at >= ? AND s.source = ? - AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", - (cutoff, source), + self._GET_TOOL_CALLS_WITH_SOURCE, (cutoff, source) ) else: - cursor2 = self._conn.execute( - """SELECT m.tool_calls - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE s.started_at >= ? - AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", - (cutoff,), - ) + cursor2 = self._conn.execute(self._GET_TOOL_CALLS_ALL, (cutoff,)) tool_calls_counts = Counter() for row in cursor2.fetchall(): @@ -294,22 +376,10 @@ def _get_skill_usage(self, cutoff: float, source: str = None) -> List[Dict]: if source: cursor = self._conn.execute( - """SELECT m.tool_calls, m.timestamp - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE s.started_at >= ? AND s.source = ? - AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", - (cutoff, source), + self._GET_SKILL_CALLS_WITH_SOURCE, (cutoff, source) ) else: - cursor = self._conn.execute( - """SELECT m.tool_calls, m.timestamp - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE s.started_at >= ? - AND m.role = 'assistant' AND m.tool_calls IS NOT NULL""", - (cutoff,), - ) + cursor = self._conn.execute(self._GET_SKILL_CALLS_ALL, (cutoff,)) for row in cursor.fetchall(): try: diff --git a/agent/interrupt_compat.py b/agent/interrupt_compat.py new file mode 100644 index 000000000000..bf56849495c7 --- /dev/null +++ b/agent/interrupt_compat.py @@ -0,0 +1,35 @@ +"""Compatibility helper for explicit agent stop producers.""" + +from __future__ import annotations + +import inspect +from typing import Any + + +def request_hard_interrupt(agent: Any, message: str | None = None) -> bool: + """Request an explicit stop, falling back to the legacy interrupt ABI. + + New agents expose ``hard_interrupt(message=None)``. Third-party agents and + old test doubles may only expose ``interrupt(message=None)``; keep those + usable without sending the newer ``hard_cancel=`` keyword they do not know. + Returns ``False`` only when neither callable is available. + """ + # Avoid treating a dynamic ``__getattr__`` proxy (notably an unspecced + # ``MagicMock`` or a third-party RPC facade) as if it genuinely implements + # the new ABI. Static lookup proves the attribute exists on the instance or + # its type before normal descriptor binding retrieves the callable. + try: + inspect.getattr_static(agent, "hard_interrupt") + except AttributeError: + interrupt = None + else: + interrupt = getattr(agent, "hard_interrupt", None) + if not callable(interrupt): + interrupt = getattr(agent, "interrupt", None) + if not callable(interrupt): + return False + if message is None: + interrupt() + else: + interrupt(message) + return True diff --git a/agent/iteration_budget.py b/agent/iteration_budget.py index 213b97c02265..7d50026c1701 100644 --- a/agent/iteration_budget.py +++ b/agent/iteration_budget.py @@ -2,7 +2,7 @@ Extracted from ``run_agent.py``. Each ``AIAgent`` instance (parent or subagent) holds an :class:`IterationBudget`; the parent's cap comes from -``max_iterations`` (default 90), each subagent's cap comes from +``max_iterations`` (default 500), each subagent's cap comes from ``delegation.max_iterations`` (default 50). ``run_agent`` re-exports ``IterationBudget`` so existing @@ -18,7 +18,7 @@ class IterationBudget: """Thread-safe iteration counter for an agent. Each agent (parent or subagent) gets its own ``IterationBudget``. - The parent's budget is capped at ``max_iterations`` (default 90). + The parent's budget is capped at ``max_iterations`` (default 500). Each subagent gets an independent budget capped at ``delegation.max_iterations`` (default 50) — this means total iterations across parent + subagents can exceed the parent's cap. diff --git a/agent/learning_graph_render.py b/agent/learning_graph_render.py index 3602ee270e61..479b2f5b4b74 100644 --- a/agent/learning_graph_render.py +++ b/agent/learning_graph_render.py @@ -403,7 +403,6 @@ def _category_counts(payload: dict[str, Any]) -> list[tuple[str, int]]: def category_color_map(payload: dict[str, Any]) -> dict[str, str]: """Deterministic, evenly-spread hue per skill category (theme-independent).""" clusters = _category_counts(payload) - n = max(1, len(clusters)) # Golden-angle hue spacing so adjacent categories never collide in color. return {cat: rgb_to_hex(_hsl_to_rgb((i * 137.508) % 360, 0.55, 0.62)) for i, (cat, _c) in enumerate(clusters)} diff --git a/agent/lsp/cli.py b/agent/lsp/cli.py index 139baa213f77..607c156d102a 100644 --- a/agent/lsp/cli.py +++ b/agent/lsp/cli.py @@ -55,7 +55,7 @@ def register_subparser(subparsers: argparse._SubParsersAction) -> None: help="Even attempt servers marked manual-install (best effort)", ) - sub_restart = sub.add_parser( + sub.add_parser( "restart", help="Tear down running LSP clients (next edit re-spawns)", ) diff --git a/agent/lsp/eventlog.py b/agent/lsp/eventlog.py index b38627504b4a..f118ccf0acea 100644 --- a/agent/lsp/eventlog.py +++ b/agent/lsp/eventlog.py @@ -40,7 +40,7 @@ import logging import os import threading -from typing import Tuple +from typing import List, Tuple # Dedicated logger name so the documented grep recipe survives a # ``logging.getLogger(__name__)`` rename of any internal module. @@ -188,6 +188,25 @@ def log_spawn_failed(server_id: str, workspace_root: str, exc: BaseException) -> ) +def log_reaped(keys: List[Tuple[str, str]], idle_timeout: float) -> None: + """Idle clients were shut down by the reaper. INFO — one line per + sweep so users can correlate memory drops with LSP activity. + + Also clears the ``log_active`` announce cache for the reaped keys so + a later respawn re-announces at INFO instead of logging a misleading + DEBUG "reused client". + """ + with _announce_lock: + for key in keys: + _announced_active.discard(key) + summary = ", ".join(f"{sid} ({root})" for sid, root in keys) + _emit( + "reaper", + logging.INFO, + f"reaped {len(keys)} idle client(s) after {idle_timeout:.0f}s: {summary}", + ) + + def reset_announce_caches() -> None: """Test-only: clear the dedup caches. Production code never calls this.""" with _announce_lock: @@ -209,5 +228,6 @@ def reset_announce_caches() -> None: "log_timeout", "log_server_error", "log_spawn_failed", + "log_reaped", "reset_announce_caches", ] diff --git a/agent/lsp/install.py b/agent/lsp/install.py index 68a9751e7fa9..fc9bea59307b 100644 --- a/agent/lsp/install.py +++ b/agent/lsp/install.py @@ -30,12 +30,12 @@ import os import shutil import subprocess -import sys import threading from pathlib import Path from typing import Any, Dict, Optional from hermes_cli._subprocess_compat import windows_hide_flags +from hermes_constants import find_node_executable logger = logging.getLogger("agent.lsp.install") @@ -124,10 +124,9 @@ def _is_windows() -> bool: def hermes_lsp_bin_dir() -> Path: """Return the Hermes-owned bin staging dir for LSP servers.""" - home = os.environ.get("HERMES_HOME") - if home is None: - home = os.path.join(os.path.expanduser("~"), ".hermes") - p = Path(home) / "lsp" / "bin" + from hermes_constants import get_hermes_home + + p = get_hermes_home() / "lsp" / "bin" p.mkdir(parents=True, exist_ok=True) return p @@ -251,9 +250,12 @@ def _install_npm( peer deps that npm doesn't auto-pull (typescript-language-server needs ``typescript`` next to it; intelephense ships standalone). """ - npm = shutil.which("npm") + # Managed npm first: $HERMES_HOME/node is not on an arbitrary process's + # PATH, so a bare which() misses the Node that Hermes installed and + # reports "npm not on PATH" on a machine that has a perfectly good one. + npm = find_node_executable("npm") if npm is None: - logger.info("[install] cannot install %s: npm not on PATH", pkg) + logger.info("[install] cannot install %s: no usable npm found", pkg) return None staging = hermes_lsp_bin_dir().parent # /lsp/ install_targets = [pkg] + list(extra_pkgs or []) @@ -267,7 +269,7 @@ def _install_npm( [npm, "install", "--prefix", str(staging), "--silent", "--no-fund", "--no-audit", *install_targets], check=False, capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=300, stdin=subprocess.DEVNULL, creationflags=windows_hide_flags(), @@ -316,7 +318,7 @@ def _install_go(pkg: str, bin_name: str) -> Optional[str]: [go, "install", pkg], check=False, capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=600, env=env, stdin=subprocess.DEVNULL, diff --git a/agent/lsp/manager.py b/agent/lsp/manager.py index d3b4244790b1..7ba1b914f74c 100644 --- a/agent/lsp/manager.py +++ b/agent/lsp/manager.py @@ -59,6 +59,7 @@ logger = logging.getLogger("agent.lsp.manager") DEFAULT_IDLE_TIMEOUT = 600 # seconds; servers idle for >10min get reaped +MIN_IDLE_TIMEOUT = 30 # floor for config values; must exceed any per-op wait budget class _BackgroundLoop: @@ -176,6 +177,7 @@ def __init__( self._spawning: Dict[Tuple[str, str], asyncio.Future] = {} self._last_used: Dict[Tuple[str, str], float] = {} self._state_lock = threading.Lock() + self._idle_reaper_task: Optional[asyncio.Task] = None # Delta baseline: file path → snapshot of diagnostics taken # immediately before a write. ``get_diagnostics_sync`` filters @@ -183,6 +185,9 @@ def __init__( # introduced by the current edit. self._delta_baseline: Dict[str, List[Dict[str, Any]]] = {} + if self._enabled and self._idle_timeout > 0: + self._loop.run(self._start_idle_reaper(), timeout=2.0) + @classmethod def create_from_config(cls) -> Optional["LSPService"]: """Build a service from ``hermes_cli.config`` settings. @@ -191,8 +196,8 @@ def create_from_config(cls) -> Optional["LSPService"]: itself returns ``is_active()`` False when LSP is disabled. """ try: - from hermes_cli.config import load_config - cfg = load_config() + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() except Exception as e: # noqa: BLE001 logger.debug("LSP config load failed: %s", e) return None @@ -205,6 +210,16 @@ def create_from_config(cls) -> Optional["LSPService"]: wait_mode = lsp_cfg.get("wait_mode", "document") wait_timeout = float(lsp_cfg.get("wait_timeout", DIAGNOSTICS_DOCUMENT_WAIT)) install_strategy = lsp_cfg.get("install_strategy", "auto") + try: + idle_timeout = float(lsp_cfg.get("idle_timeout", DEFAULT_IDLE_TIMEOUT)) + except (TypeError, ValueError): + idle_timeout = DEFAULT_IDLE_TIMEOUT + if 0 < idle_timeout < MIN_IDLE_TIMEOUT: + # A timeout below the per-operation wait budget could reap a + # client mid-flight; the resulting outer timeout would then + # mark the (server, workspace) pair broken for the process + # lifetime. Clamp to a safe floor (0 still disables). + idle_timeout = MIN_IDLE_TIMEOUT servers_cfg = lsp_cfg.get("servers") or {} disabled = [] binary_overrides: Dict[str, List[str]] = {} @@ -235,6 +250,7 @@ def create_from_config(cls) -> Optional["LSPService"]: env_overrides=env_overrides, init_overrides=init_overrides, disabled_servers=disabled, + idle_timeout=idle_timeout, ) # ------------------------------------------------------------------ @@ -434,6 +450,7 @@ def _mark_broken_for_file(self, file_path: str, exc: BaseException) -> None: # ``_clients`` with a half-initialized state. with self._state_lock: client = self._clients.pop(key, None) + self._last_used.pop(key, None) if client is not None: try: # Fire-and-forget shutdown — give it a second to cleanup, @@ -470,7 +487,7 @@ async def _snapshot_async(self, file_path: str) -> List[Dict[str, Any]]: except Exception as e: # noqa: BLE001 logger.debug("snapshot open/wait failed: %s", e) return [] - self._last_used[(client.server_id, client.workspace_root)] = time.time() + self._touch(client) if not fresh: # No fresh data for the pre-edit content — an empty baseline # is safe: worst case the delta filter removes less, never @@ -499,7 +516,7 @@ async def _open_and_wait_async(self, file_path: str) -> Optional[List[Dict[str, except Exception as e: # noqa: BLE001 logger.debug("open/wait failed for %s: %s", file_path, e) return None - self._last_used[(client.server_id, client.workspace_root)] = time.time() + self._touch(client) if not fresh: return None return list(client.diagnostics_for(file_path, fresh_only=True)) @@ -539,6 +556,7 @@ async def _get_or_spawn(self, file_path: str) -> Optional[LSPClient]: with self._state_lock: client = self._clients.get(key) if client is not None and client.is_running: + self._last_used[key] = time.time() eventlog.log_active(srv.server_id, per_server_root) return client spawning = self._spawning.get(key) @@ -589,7 +607,7 @@ async def _get_or_spawn(self, file_path: str) -> Optional[LSPClient]: return None with self._state_lock: self._clients[key] = client - self._last_used[key] = time.time() + self._last_used[key] = time.time() eventlog.log_active(srv.server_id, per_server_root) spawn_future.set_result(client) return client @@ -597,7 +615,63 @@ async def _get_or_spawn(self, file_path: str) -> Optional[LSPClient]: with self._state_lock: self._spawning.pop(key, None) + async def _start_idle_reaper(self) -> None: + self._idle_reaper_task = asyncio.create_task(self._idle_reaper_loop()) + + def _touch(self, client: LSPClient) -> None: + """Refresh the last-used timestamp for a client we just used. + + Guarded on membership so a reaped-mid-operation client can't + resurrect an orphan ``_last_used`` entry after the reaper popped + the key. All writers and the reaper run on the background loop + thread; the lock keeps this consistent with the reader anyway. + """ + key = (client.server_id, client.workspace_root) + with self._state_lock: + if key in self._clients: + self._last_used[key] = time.time() + + async def _idle_reaper_loop(self) -> None: + interval = min(60.0, self._idle_timeout) + while True: + await asyncio.sleep(interval) + try: + await self._reap_idle_once() + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 + # A transient sweep error must not kill the reaper — + # otherwise one bad shutdown permanently re-opens the + # unbounded-accumulation leak this loop exists to fix. + logger.debug("LSP idle reaper sweep error: %s", e) + + async def _reap_idle_once(self) -> None: + cutoff = time.time() - self._idle_timeout + with self._state_lock: + idle_keys = [ + key + for key in self._clients + if self._last_used.get(key, 0) < cutoff + ] + clients = [self._clients.pop(key) for key in idle_keys] + for key in idle_keys: + self._last_used.pop(key, None) + if clients: + eventlog.log_reaped( + [(c.server_id, c.workspace_root) for c in clients], + self._idle_timeout, + ) + await asyncio.gather( + *(client.shutdown() for client in clients), + return_exceptions=True, + ) + async def _shutdown_async(self) -> None: + reaper = self._idle_reaper_task + self._idle_reaper_task = None + if reaper is not None: + reaper.cancel() + await asyncio.gather(reaper, return_exceptions=True) with self._state_lock: clients = list(self._clients.values()) self._clients.clear() diff --git a/agent/lsp/servers.py b/agent/lsp/servers.py index 4056ba4dbab6..fc2a0b261693 100644 --- a/agent/lsp/servers.py +++ b/agent/lsp/servers.py @@ -710,9 +710,9 @@ def _find_pses_bundle(ctx: ServerContext) -> Optional[str]: env_path = os.environ.get("PSES_BUNDLE_PATH") if env_path: candidates.append(env_path) - home = os.environ.get("HERMES_HOME") or os.path.join( - os.path.expanduser("~"), ".hermes" - ) + from hermes_constants import get_hermes_home + + home = str(get_hermes_home()) candidates.append(os.path.join(home, "lsp", "PowerShellEditorServices")) for cand in candidates: @@ -796,9 +796,9 @@ def _spawn_powershell_es(root: str, ctx: ServerContext) -> Optional[SpawnSpec]: def hermes_lsp_session_dir() -> str: """Return (and create) the dir for PSES session/log scratch files.""" - home = os.environ.get("HERMES_HOME") or os.path.join( - os.path.expanduser("~"), ".hermes" - ) + from hermes_constants import get_hermes_home + + home = str(get_hermes_home()) d = os.path.join(home, "lsp", "pses") os.makedirs(d, exist_ok=True) return d diff --git a/agent/memory_provider.py b/agent/memory_provider.py index 4210a4c252e5..559fc3df6c88 100644 --- a/agent/memory_provider.py +++ b/agent/memory_provider.py @@ -34,12 +34,50 @@ from __future__ import annotations import logging +import re from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) +# Prompts that carry no semantic signal — trivial acknowledgements, greetings, +# slash commands, empty input. Single source of truth shared by the core +# per-turn prefetch gate (agent/turn_context.py, run_agent.py) and provider- +# side classifiers (plugins/memory/honcho) so the two can never drift apart. +# The alternation is anchored and may only be followed by whitespace or +# punctuation, so words that merely START with a trivial word ("k8s", "yolo", +# "note", "hindsight") do NOT match, while trailing-punctuation variants +# ("hi!", "hey.", "thanks :)", "done???") do. +TRIVIAL_PROMPT_RE = re.compile( + r'^(yes|no|ok|okay|sure|thanks|thank you|y|n|yep|nope|yeah|nah|' + r'hi|hey|hello|yo|sup|' + r'continue|go ahead|do it|proceed|got it|cool|nice|great|done|next|lgtm|k)' + r'[\s!?.:;,"' + "'" + r'~\u2018\u2019\u201c\u201d\u2014\u2013\u2026()\[\]{}<>*&^%$#@!+=`\u00a0]*$', + re.IGNORECASE, +) + + +def is_trivial_prompt(text: Optional[str]) -> bool: + """Return True if a user prompt is too trivial to warrant memory recall. + + Empty/whitespace-only input, slash commands, and bare greetings or + acknowledgements (with optional trailing punctuation) all count as + trivial. Callers use this to skip memory-provider prefetch/injection + on turns that carry no semantic signal — saving a blocking network + round-trip and preventing stale user-model context from derailing + one-word replies. + """ + if not text: + return True + stripped = text.strip() + if not stripped: + return True + if stripped.startswith("/"): + return True + return bool(TRIVIAL_PROMPT_RE.match(stripped)) + + class MemoryProvider(ABC): """Abstract base class for memory providers.""" @@ -253,6 +291,10 @@ def get_config_schema(self) -> List[Dict[str, Any]]: required: True if required (default: False) default: default value (optional) choices: list of valid values (optional) + type: text, integer, number, or boolean (optional) + minimum: numeric lower bound for integer/number fields (optional) + maximum: numeric upper bound for integer/number fields (optional) + step: numeric input step for Dashboard rendering (optional) url: URL where user can get this credential (optional) env_var: explicit env var name for secrets (default: auto-generated) diff --git a/agent/message_sanitization.py b/agent/message_sanitization.py index 29a4b8691ae8..dc4df3dd2702 100644 --- a/agent/message_sanitization.py +++ b/agent/message_sanitization.py @@ -14,6 +14,7 @@ from __future__ import annotations +import hashlib import json import logging import re @@ -474,4 +475,378 @@ def _walk(node): "_sanitize_tools_non_ascii", "_strip_images_from_messages", "_sanitize_structure_non_ascii", + # call_id policy owners (F4 consolidation) + "deterministic_call_id", + "coalesce_tool_call_id", + "uniquify_tool_call_ids", + # reasoning_content policy owners (F4 consolidation) + "reasoning_echo_family", + "matches_reasoning_echo_family", + "needs_reasoning_echo", + "apply_reasoning_content_policy", + "reapply_reasoning_echo", ] + + +# --------------------------------------------------------------------------- +# call_id policy — single owner (audit F4, incident chain I4) +# --------------------------------------------------------------------------- +# +# Three forked policy sites converged here: +# * agent/codex_responses_adapter.py `_deterministic_call_id` — hash +# synthesis when a provider omits call_id (fa3ab2ffd0 → e45f2b39e2). +# * run_agent.AIAgent._get_tool_call_id_static — `call_id or id` +# coalescing for dicts and SDK objects. +# * run_agent.AIAgent._uniquify_tool_call_ids — duplicate-id repair with +# deterministic `_d` suffixes (#58327 loss class). +# +# NOT consolidated (different scheme on purpose): +# agent/transports/codex_event_projector._deterministic_call_id maps codex +# app-server ITEM ids (`codex__`), not chat tool-call +# content; merging the two would change ids and invalidate prompt caches. +# +# HARD INVARIANT: everything here must stay deterministic (never uuid4) and +# byte-identical for existing inputs — these ids feed prompt-cache prefixes. + + +def deterministic_call_id(fn_name: str, arguments: str, index: int = 0) -> str: + """Generate a deterministic call_id from tool call content. + + Used as a fallback when the API doesn't provide a call_id. + Deterministic IDs prevent cache invalidation — random UUIDs would + make every API call's prefix unique, breaking OpenAI's prompt cache. + """ + seed = f"{fn_name}:{arguments}:{index}" + digest = hashlib.sha256(seed.encode("utf-8", errors="replace")).hexdigest()[:12] + return f"call_{digest}" + + +def coalesce_tool_call_id(tc: Any) -> str: + """Extract the effective call ID from a tool_call entry (dict or object). + + Single owner for the ``call_id or id`` coalescing rule: Codex Responses + tool calls carry ``call_id`` (authoritative pairing key), Chat + Completions ones carry ``id`` only. Returns ``""`` when neither is set. + """ + if isinstance(tc, dict): + return (tc.get("call_id", "") or tc.get("id", "") or "").strip() + return (getattr(tc, "call_id", "") or getattr(tc, "id", "") or "").strip() + + +def uniquify_tool_call_ids(tool_calls: list) -> list: + """Ensure every tool call in a single assistant turn has a distinct id. + + Some models/providers reuse one call id across different calls in a + single batch (observed with native Kimi Responses replays, Ollama- + compatible endpoints, and degraded models at long context; same bug + class as openclaw/openclaw#110518 / #110956). Duplicate ids are lossy + downstream: the pre-API sanitizer keeps only the first call/result + pair per id (#58327), so the later call's result silently vanishes + from every replayed payload, and strict providers (Anthropic + tool_use, DeepSeek) reject duplicate ids outright. + + The first occurrence keeps its id; later collisions get a + deterministic ``_d`` suffix — never a random UUID, which would + break prompt-cache prefix stability across replays. Mutates the + entries in place (SDK models / SimpleNamespace / dicts) and returns + the same list. Blank/missing ids are left for the deterministic + fallback in ``build_assistant_message``. + """ + seen: set = set() + for tc in tool_calls or []: + # Same coalescing rule as ``coalesce_tool_call_id`` but tolerant of + # non-string ids (degraded models can emit ints/None here). + if isinstance(tc, dict): + raw = tc.get("call_id") or tc.get("id") or "" + else: + raw = getattr(tc, "call_id", None) or getattr(tc, "id", None) or "" + raw = raw.strip() if isinstance(raw, str) else "" + if not raw: + continue + # Composite Responses ids ("call_x|fc_y") collide on the call + # half — that's the pairing key providers enforce per turn. + cid = raw.split("|", 1)[0] + if not cid: + continue + if cid not in seen: + seen.add(cid) + continue + n = 2 + new_id = f"{cid}_d{n}" + while new_id in seen: + n += 1 + new_id = f"{cid}_d{n}" + seen.add(new_id) + + def _renamed(value): + # Preserve a composite id's response-item half so the + # provider's real fc_/item id survives the rename. + if isinstance(value, str) and "|" in value: + return f"{new_id}|{value.split('|', 1)[1]}" + return new_id + + try: + if isinstance(tc, dict): + if tc.get("id"): + tc["id"] = _renamed(tc["id"]) + else: + tc["id"] = new_id + if tc.get("call_id"): + tc["call_id"] = new_id + else: + tc.id = _renamed(getattr(tc, "id", None)) + if getattr(tc, "call_id", None): + tc.call_id = new_id + except Exception: + logger.warning( + "Could not uniquify duplicate tool call id %s", cid + ) + continue + _fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) + _fn_name = (_fn.get("name") if isinstance(_fn, dict) else getattr(_fn, "name", None)) or "?" + logger.warning( + "Model reused tool call id %s within one turn; renamed the " + "duplicate to %s (tool=%s) to keep call/result pairing " + "lossless.", cid, new_id, _fn_name, + ) + return tool_calls + + +# --------------------------------------------------------------------------- +# reasoning_content policy — single owner (audit F4) +# --------------------------------------------------------------------------- +# +# The strip-vs-repad decision was previously forked across the wire files in +# separate incident commits (2b3a4f0af8 strip for strict providers, +# b5495db701 re-pad for require-side, 94b3131be7/9a9f8a6d99 kimi pad). The +# POLICY — which provider direction gets which treatment — lives here as one +# rule table + apply functions; adapters keep only SYNTAX mapping (e.g. +# anthropic_adapter turning reasoning_content into a thinking block). +# +# Direction table: +# require-side (echo-back enforced; replays 400 without the field): +# kimi — provider kimi-coding/kimi-coding-cn, or host api.kimi.com / +# moonshot.ai / moonshot.cn. Host-driven on purpose: +# aggregators re-exporting kimi models reject the echo. +# deepseek — provider "deepseek", model contains "deepseek", or host +# api.deepseek.com (#15250; V4 rejects empty-string pads, +# hence the " " single-space pad, #17341). +# mimo — provider "xiaomi", model contains "mimo", or host +# *.xiaomimimo.com. +# strict side (field rejected with 400/422 "Extra inputs are not +# permitted"): everyone else — Mistral, Cerebras, Groq, SambaNova, … +# (#45655). Strip the key entirely, even a single-space pad. + +_REASONING_ECHO_RULES: tuple = ( + # (family, exact providers (raw), exact providers (lowered), + # model substrings (lowered), base_url hosts) + ("kimi", frozenset({"kimi-coding", "kimi-coding-cn"}), frozenset(), (), + ("api.kimi.com", "moonshot.ai", "moonshot.cn")), + ("deepseek", frozenset(), frozenset({"deepseek"}), ("deepseek",), + ("api.deepseek.com",)), + ("mimo", frozenset(), frozenset({"xiaomi"}), ("mimo",), + ("api.xiaomimimo.com", "xiaomimimo.com")), +) + + +def _family_rule(family: str) -> tuple: + for rule in _REASONING_ECHO_RULES: + if rule[0] == family: + return rule + raise KeyError(family) + + +def matches_reasoning_echo_family( + family: str, provider: Any, model: Any, base_url: Any +) -> bool: + """True when (provider, model, base_url) matches one echo-back family. + + Families can overlap (e.g. a deepseek-named model pointed at a kimi + host); this membership test is independent per family so per-family + predicates keep their original semantics. + """ + from utils import base_url_host_matches + + _, raw_providers, lowered_providers, model_subs, hosts = _family_rule(family) + provider_lower = (provider or "").lower() + model_lower = (model or "").lower() + if provider in raw_providers or provider_lower in lowered_providers: + return True + if any(sub in model_lower for sub in model_subs): + return True + return any(base_url_host_matches(base_url, host) for host in hosts) + + +def reasoning_echo_family(provider: Any, model: Any, base_url: Any) -> "str | None": + """Classify the provider direction for the reasoning_content echo policy. + + Returns ``"kimi"``, ``"deepseek"``, or ``"mimo"`` (first match in table + order) when the target endpoint enforces reasoning_content echo-back on + assistant turns, else ``None`` (strict/indifferent side — the field must + be stripped). + """ + for rule in _REASONING_ECHO_RULES: + if matches_reasoning_echo_family(rule[0], provider, model, base_url): + return rule[0] + return None + + +def needs_reasoning_echo(provider: Any, model: Any, base_url: Any) -> bool: + """True when the endpoint requires reasoning_content echo-back.""" + return reasoning_echo_family(provider, model, base_url) is not None + + +def apply_reasoning_content_policy( + source_msg: dict, api_msg: dict, needs_thinking_pad: bool +) -> None: + """Copy provider-facing reasoning fields onto an API replay message. + + ``needs_thinking_pad`` is the require-side flag (see + ``needs_reasoning_echo`` / the agent's cached + ``_needs_thinking_reasoning_pad``). Mutates ``api_msg`` in place. + """ + if source_msg.get("role") != "assistant": + return + + # 1. Explicit reasoning_content already set. + # + # When the active provider enforces the thinking-mode echo-back + # (DeepSeek / Kimi / MiMo), preserve it verbatim — that includes their + # own space-placeholder written at creation time and any valid reasoning + # from the same provider. Sessions persisted BEFORE #17341 have + # empty-string placeholders pinned at creation time; DeepSeek V4 Pro + # rejects those with HTTP 400, so upgrade "" → " " on replay. + # + # When the active provider does NOT enforce echo-back, strip the field + # entirely. Strict OpenAI-compatible providers (Mistral, Cerebras, Groq, + # SambaNova, …) reject ANY reasoning_content key in input messages with + # HTTP 400/422 ("Extra inputs are not permitted"), even an empty string + # or a single-space pad. This is the cross-provider fallback case: a + # reasoning primary (DeepSeek/Kimi/MiMo) pads history with " ", then a + # fallback to a strict provider replays that pad and 422s. Stripping + # here covers the rebuild path; ``reapply_reasoning_echo`` covers the + # already-built api_messages path. Refs #45655. + existing = source_msg.get("reasoning_content") + if isinstance(existing, str): + if not needs_thinking_pad: + api_msg.pop("reasoning_content", None) + elif existing == "": + api_msg["reasoning_content"] = " " + else: + api_msg["reasoning_content"] = existing + return + + # 2. Cross-provider poisoned history (#15748): on DeepSeek/Kimi, + # if the source turn has tool_calls AND a 'reasoning' field but no + # 'reasoning_content' key, the 'reasoning' text was written by a + # prior provider (e.g. MiniMax) — DeepSeek's own _build_assistant_message + # pins reasoning_content at creation time for tool-call turns, so the + # shape (reasoning set, reasoning_content absent, tool_calls present) + # is unreachable from same-provider DeepSeek history after this fix. + # Inject a single space to satisfy the API without leaking another + # provider's chain of thought to DeepSeek/Kimi. Space (not "") + # because DeepSeek V4 Pro rejects empty-string reasoning_content + # in thinking mode (refs #17341). + normalized_reasoning = source_msg.get("reasoning") + if ( + needs_thinking_pad + and source_msg.get("tool_calls") + and isinstance(normalized_reasoning, str) + and normalized_reasoning + ): + api_msg["reasoning_content"] = " " + return + + # 3. Healthy session: promote 'reasoning' field to 'reasoning_content' + # for providers that use the internal 'reasoning' key. + # This must happen before the unconditional empty-string fallback so + # genuine reasoning content is not overwritten (#15812 regression in + # PR #15478). Only promote for providers that enforce echo-back — + # strict providers reject the field (refs #45655). + if isinstance(normalized_reasoning, str) and normalized_reasoning: + if needs_thinking_pad: + api_msg["reasoning_content"] = normalized_reasoning + else: + api_msg.pop("reasoning_content", None) + return + + # 4. DeepSeek / Kimi thinking mode: all assistant messages need + # reasoning_content. Inject a single space to satisfy the provider's + # requirement when no explicit reasoning content is present. Covers + # both tool-call turns (already-poisoned history with no reasoning + # at all) and plain text turns. Space (not "") because DeepSeek V4 + # Pro tightened validation and rejects empty string with HTTP 400 + # ("The reasoning content in the thinking mode must be passed back + # to the API"). Refs #17341. + if needs_thinking_pad: + api_msg["reasoning_content"] = " " + return + + # 5. reasoning_content was present but not a string (e.g. None after + # context compaction). Don't pass null to the API. + api_msg.pop("reasoning_content", None) + + +def reapply_reasoning_echo(api_messages: list, needs_thinking_pad: bool) -> int: + """Re-pad (or strip) assistant turns' reasoning_content for the active provider. + + ``api_messages`` is built once, before the retry loop, while the *primary* + provider is active. A mid-conversation fallback can then switch providers, + so the reasoning fields baked into ``api_messages`` are shaped for the + *prior* provider and must be reconciled against the *current* one: + + * Switching TO a require-side provider (DeepSeek / Kimi / MiMo thinking + mode): assistant turns built when the prior provider did NOT need the + echo-back go out without ``reasoning_content`` and the new provider + rejects them with HTTP 400 ("The reasoning_content in the thinking mode + must be passed back"). Re-apply the pad. + + * Switching TO a strict provider that rejects the field (Mistral, + Cerebras, Groq, SambaNova, …): assistant turns built under a reasoning + primary carry a ``reasoning_content`` pad (often a single space ``" "``), + and the strict provider rejects it with HTTP 400/422 ("Extra inputs are + not permitted"). Strip the field. This is the exact cross-provider + fallback bug from #45655 — a DeepSeek primary pads history with ``" "``, + the request falls back to Mistral, and Mistral 422s on the stale pad. + + Calling this immediately before building the request kwargs reconciles the + fields against the *current* provider. It is idempotent and safe to call + every iteration; it covers every fallback path. + + Returns the number of assistant turns whose reasoning_content was added or + removed. + """ + changed = 0 + for api_msg in api_messages: + if api_msg.get("role") != "assistant": + continue + if needs_thinking_pad: + if api_msg.get("reasoning_content"): + continue + apply_reasoning_content_policy(api_msg, api_msg, needs_thinking_pad) + if api_msg.get("reasoning_content"): + changed += 1 + else: + # Strict provider — strip any stale reasoning_content pad left + # over from a reasoning primary so the fallback request doesn't + # 400/422 on it. + if "reasoning_content" in api_msg: + api_msg.pop("reasoning_content", None) + changed += 1 + return changed + + +# --------------------------------------------------------------------------- +# Image / multimodal parts — evaluated, NOT consolidated (verdict: syntax) +# --------------------------------------------------------------------------- +# +# The per-adapter image handling is format-specific SYNTAX, not shared policy: +# * anthropic_adapter (~1817): data-URL → Anthropic `source: {type: base64}` +# block mapping — Anthropic wire shape only. +# * codex_responses_adapter (~113/165/812): chat `image_url` parts → +# Responses `input_image` items and image counting for log summaries — +# Responses wire shape only. +# * transports/chat_completions: pass-through (native format). +# The one genuinely shared image POLICY — removing images when a server +# rejects them while preserving tool_call_id pairing — already has a single +# owner here: ``_strip_images_from_messages`` above. diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 21fae396d9dd..26e9523ec34d 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -13,6 +13,7 @@ import re import threading from concurrent.futures import ThreadPoolExecutor, wait as _futures_wait +from types import SimpleNamespace from typing import Any from agent.auxiliary_client import call_llm @@ -382,30 +383,42 @@ def _merge_slot_extra_body( def _maybe_apply_moa_cache_control( messages: list[dict[str, Any]], runtime: dict[str, Any], + *, + cache_disabled: bool | None = None, ) -> list[dict[str, Any]]: """Decorate an advisor or aggregator request with cache_control when its route honors it. Reuses the SAME policy function as the main agent loop (``anthropic_prompt_cache_policy``) resolved against the slot's own - provider/base_url/api_mode/model, and the SAME breakpoint layout - (``apply_anthropic_cache_control``, system_and_3). This keeps advisor and - aggregator calls decorated exactly like an acting agent on that provider - would be — no MoA-specific caching logic to drift. + provider/base_url/api_mode/model and shared marker helper + (``apply_anthropic_cache_control``). MoA has no per-session static prefix, + so it uses the helper's legacy system-and-3 fallback without carrying a + separate caching strategy. Returns the messages unchanged on any resolution error or when the policy says the route doesn't honor markers. + + ``cache_disabled`` (or the live config when omitted) is stamped onto the + policy stub so ``prompt_caching.cache_ttl: off`` is not bypassed by the + blank-agent pattern (#76085). """ try: - from types import SimpleNamespace - - from agent.agent_runtime_helpers import anthropic_prompt_cache_policy + from agent.agent_runtime_helpers import ( + anthropic_prompt_cache_policy, + blank_cache_policy_stub, + ) from agent.prompt_caching import apply_anthropic_cache_control + # Prefer an explicit kwarg, then a snapshot on the runtime dict + # (threaded from the live agent), else config via the stub factory. + if cache_disabled is None and "_cache_disabled" in runtime: + cache_disabled = runtime.get("_cache_disabled") + # The policy function reads agent.* only as fallbacks for kwargs we - # don't pass; provide a stub so the slot is judged purely on its own - # resolved runtime. - stub = SimpleNamespace(provider="", base_url="", api_mode="", model="") + # don't pass; blank_cache_policy_stub is the only sanctioned stub + # so _cache_disabled cannot be left off again (#76085). + stub = blank_cache_policy_stub(cache_disabled) should_cache, native_layout = anthropic_prompt_cache_policy( stub, provider=runtime.get("provider") or "", @@ -431,6 +444,7 @@ def _run_reference( max_tokens: int | None = None, reference_timeout: float | None = None, context_length_cache: Any = None, + cache_disabled: bool | None = None, ) -> tuple[str, str, Any]: """Call one reference model and return ``(label, text, accounting)``. @@ -480,10 +494,11 @@ def _run_reference( reserve_output_tokens=max_tokens, context_length_cache=context_length_cache, ) - # Apply the same Anthropic-style prompt-caching decoration the main - # agent loop applies (system_and_3 breakpoints). The advisory view is - # append-only across iterations (new turns append before the trailing - # synthetic marker), so on cache-honoring routes (Claude via + # Apply the Anthropic-style prompt-caching decoration used by the main + # agent loop. This fixed reference prompt has no session-specific + # prefix split, so the helper uses its legacy system-and-3 fallback. + # The advisory view is append-only across iterations (new turns append + # before the trailing synthetic marker), so on cache-honoring routes (Claude via # OpenRouter/native, MiniMax, Qwen/DashScope) iteration N+1's prefix # replays iteration N's cached prefix. Without this, Claude advisors # served ZERO cache reads across an entire benchmark run (measured: @@ -491,7 +506,12 @@ def _run_reference( # caching is opt-in per request. OpenAI-family advisors are untouched # (their caching is automatic; markers are ignored harmlessly, but we # only decorate when the policy says the route honors them). - messages = _maybe_apply_moa_cache_control(messages, runtime) + # Pin the live agent disable onto the runtime so advisor decoration + # tracks conversation state, not a fresh config re-read (#76085). + cache_runtime = runtime + if cache_disabled is not None: + cache_runtime = {**runtime, "_cache_disabled": cache_disabled} + messages = _maybe_apply_moa_cache_control(messages, cache_runtime) # Per-slot max_tokens takes precedence over the preset-level # reference_max_tokens passed in by the caller. This lets each # reference model have its own output cap independently. @@ -795,6 +815,9 @@ def _run_references_parallel( # instead of re-probing metadata sources per reference (dict get/set is # GIL-atomic; a rare duplicate probe on a first-use race is harmless). _ctx_len_cache: dict[tuple[str, str], int | None] = {} + cache_disabled = ( + getattr(agent, "_cache_disabled", None) if agent is not None else None + ) try: for idx, slot in enumerate(reference_models): if slot.get("provider") == "moa": @@ -813,6 +836,7 @@ def _run_references_parallel( max_tokens=max_tokens, reference_timeout=reference_timeout, context_length_cache=_ctx_len_cache, + cache_disabled=cache_disabled, ) ] = idx @@ -1260,6 +1284,19 @@ def aggregate_moa_context( agg_label = _slot_label(aggregator) agg_runtime = _slot_runtime(aggregator) + # Pin the live agent disable onto synthesis decoration so mid-session + # config flips cannot re-enable markers on this path alone (#76085). + # Same not-None guard as _run_reference: stamping None would be a no-op + # (present-None falls through to the config fallback anyway). + agg_cache_runtime = agg_runtime + _agg_cache_disabled = ( + getattr(agent, "_cache_disabled", None) if agent is not None else None + ) + if _agg_cache_disabled is not None: + agg_cache_runtime = { + **agg_runtime, + "_cache_disabled": _agg_cache_disabled, + } try: # Same cache_control decoration as _run_reference's advisor calls # (see _maybe_apply_moa_cache_control) — this synthesis call is a @@ -1272,7 +1309,7 @@ def aggregate_moa_context( # breakpoints, even when the resolved aggregator slot is a # cache-honoring route (e.g. Claude on OpenRouter/native Anthropic). agg_messages = _maybe_apply_moa_cache_control( - [{"role": "user", "content": synth_prompt}], agg_runtime + [{"role": "user", "content": synth_prompt}], agg_cache_runtime ) response = call_llm( task="moa_aggregator", @@ -1299,6 +1336,53 @@ def aggregate_moa_context( ) +def _completed_response_as_stream_chunk(response: Any) -> Any: + """Convert a completed Chat Completions response into one delta stream chunk. + + MoA's outer streaming consumer expects ``choices[0].delta`` chunks. A + completed aggregator response carries ``choices[0].message`` instead; adapt + it here, at the MoA facade boundary, so provider-specific Relay behavior and + other transports remain untouched. + """ + + choices = getattr(response, "choices", None) + first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None + message = getattr(first_choice, "message", None) + raw_tool_calls = getattr(message, "tool_calls", None) + tool_call_deltas = None + if isinstance(raw_tool_calls, (list, tuple)) and raw_tool_calls: + tool_call_deltas = [] + for index, tc in enumerate(raw_tool_calls): + function = getattr(tc, "function", None) + tool_call_deltas.append(SimpleNamespace( + index=getattr(tc, "index", index), + id=getattr(tc, "id", None), + type=getattr(tc, "type", None) or "function", + function=SimpleNamespace( + name=getattr(function, "name", None), + arguments=getattr(function, "arguments", None), + ), + )) + delta = SimpleNamespace( + content=getattr(message, "content", None), + tool_calls=tool_call_deltas, + reasoning_content=getattr(message, "reasoning_content", None), + reasoning=getattr(message, "reasoning", None), + reasoning_details=getattr(message, "reasoning_details", None), + ) + choice = SimpleNamespace( + index=getattr(first_choice, "index", 0), + delta=delta, + finish_reason=getattr(first_choice, "finish_reason", None) or "stop", + ) + return SimpleNamespace( + id=getattr(response, "id", None), + model=getattr(response, "model", None), + choices=[choice], + usage=getattr(response, "usage", None), + ) + + def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str) -> None: """Attach the per-turn reference block at the END of the aggregator prompt. @@ -1336,6 +1420,63 @@ def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str agg_messages.append({"role": "user", "content": guidance}) +def peel_reference_guidance( + messages: list[dict[str, Any]], + guidance: Any, +) -> list[dict[str, Any]]: + """Remove reference guidance previously attached by ``_attach_reference_guidance``. + + Exact inverse of the three attach shapes above (string merge, trailing + text part, appended user message) — kept adjacent so the two evolve + together; a drifting separator or shape would make the peel silently + no-op and let a cache breakpoint land on the turn-varying guidance + block (the bug class #72626 fixes). + + Used by the failover redecoration chokepoint: redecoration must run on + the base transcript so the last cache breakpoint does not land on the + guidance; callers then rebase via ``rebase_prepared_request``. + + Returns a new list (input list and its messages are not mutated). + """ + if not guidance or not messages: + return messages + guidance_text = str(guidance) + last = messages[-1] + if not isinstance(last, dict) or last.get("role") != "user": + return messages + content = last.get("content") + if content == guidance_text: + # Attach shape (c): guidance was appended as its own user message. + return list(messages[:-1]) + suffix = "\n\n" + guidance_text + if isinstance(content, str) and content.endswith(suffix): + # Attach shape (a): merged into a trailing string user turn. + peeled = dict(last) + peeled["content"] = content[: -len(suffix)] + return [*messages[:-1], peeled] + if isinstance(content, list) and content: + last_part = content[-1] + if isinstance(last_part, dict) and last_part.get("type", "text") == "text": + text = last_part.get("text") or "" + if text == suffix or text == guidance_text: + # Attach shape (b): guidance rode as its own trailing part. + peeled = dict(last) + peeled["content"] = list(content[:-1]) + if not peeled["content"]: + # The guidance part was the only content — mirror the + # string shape (c) and drop the whole message rather + # than leaving an empty-content user turn behind. + return list(messages[:-1]) + return [*messages[:-1], peeled] + if text.endswith(suffix): + new_part = dict(last_part) + new_part["text"] = text[: -len(suffix)] + peeled = dict(last) + peeled["content"] = [*content[:-1], new_part] + return [*messages[:-1], peeled] + return messages + + class MoAChatCompletions: """OpenAI-chat-compatible facade where the aggregator is the acting model.""" @@ -1551,6 +1692,52 @@ def _call_prepared_aggregator( max_tokens: Any = agg_kwargs.get("max_tokens") tools: Any = agg_kwargs.get("tools") extra_body: Any = agg_kwargs.get("extra_body") + agg_runtime = _slot_runtime(aggregator) + try: + from agent.agent_runtime_helpers import ( + plan_cache_sections_for_destination, + ) + + guidance = prepared.get("guidance") + planning_messages = agg_messages + if guidance: + planning_messages = peel_reference_guidance( + agg_messages, + str(guidance), + ) + # plan_cache_sections_for_destination never mutates its inputs + # and always returns request-local copies, so the prepared + # state stays canonical. + # Tri-state: only pass a bool when a live agent snapshot exists. + # Prepared-aggregator facades built via __new__ have no _agent; + # getattr(self._agent, ...) raises and bool(None-agent) would + # force False and suppress the planner's config fallback (#76085). + _agent = getattr(self, "_agent", None) + _cache_disabled = ( + getattr(_agent, "_cache_disabled", None) + if _agent is not None + else None + ) + agg_messages, tools = plan_cache_sections_for_destination( + planning_messages, + tools, + provider=agg_runtime.get("provider") or "", + base_url=agg_runtime.get("base_url") or "", + api_mode=agg_runtime.get("api_mode") or "", + model=agg_runtime.get("model") or "", + cache_disabled=_cache_disabled, + ) + if guidance: + _attach_reference_guidance(agg_messages, str(guidance)) + except Exception as exc: # pragma: no cover - cache planning must not block MoA + # Warning, not debug: since the call-block site skips MoA, this + # block is the aggregator's ONLY decoration path — a silent + # failure here ships an undecorated request and regresses the + # exact 0%-cache MoA failure the planning exists to prevent. + logger.warning( + "MoA aggregator cache plan failed — sending undecorated " + "request (cache misses expected): %s", exc, + ) # Record the exact aggregator INPUT (incl. the injected reference # context) into the pending trace so a trace captures what the # aggregator actually saw, not a reconstruction. Traces are a @@ -1591,7 +1778,6 @@ def _call_prepared_aggregator( # actually governs the aggregator stream, not just call_llm's default. if api_kwargs.get("timeout") is not None: stream_kwargs["timeout"] = api_kwargs["timeout"] - agg_runtime = _slot_runtime(aggregator) # _slot_runtime may carry the provider's request_overrides.extra_body; # pop it and merge with the caller's extra_body (caller wins) so the # explicit kwarg below never collides with **agg_runtime. @@ -1627,6 +1813,14 @@ def _call_prepared_aggregator( self._pending_trace["aggregator_output"] = _extract_text(_agg_response) except Exception: # pragma: no cover - defensive self._pending_trace["aggregator_output"] = None + if stream and hasattr(_agg_response, "choices"): + # Some aggregator adapters (notably openai-codex Responses) consume + # their provider stream internally and return a completed response + # object even when the acting consumer requested token streaming. + # The outer chat-completions streaming loop expects delta chunks; + # hand it a one-chunk iterator instead of letting it iterate the + # SimpleNamespace response itself (#55933). + return iter((_completed_response_as_stream_chunk(_agg_response),)) return _agg_response def create(self, **api_kwargs: Any) -> Any: @@ -2116,8 +2310,24 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: except Exception: pass + resolved_preset = preset_name + if resolved_preset is None and getattr(agent, "provider", None) == "moa": + resolved_preset = getattr(agent, "model", None) + + resolved_preset = str(resolved_preset or "default") + try: + from hermes_cli.config import load_config + from hermes_cli.moa_config import normalize_moa_config + + moa_cfg = normalize_moa_config(load_config().get("moa") or {}) + presets = moa_cfg.get("presets") or {} + if resolved_preset not in presets: + resolved_preset = moa_cfg.get("default_preset") or "default" + except Exception: + resolved_preset = "default" + return MoAClient( - str(preset_name or getattr(agent, "model", None) or "default"), + resolved_preset, reference_callback=_moa_reference_relay, # Thread the agent through so the reference fan-out wait can be # aborted on a user interrupt (see _run_references_parallel). diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 288083628e02..b1701c32e89f 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -13,18 +13,40 @@ import re import time from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse -import requests import yaml +if TYPE_CHECKING: # pragma: no cover — runtime import is lazy (see below) + import requests + from utils import atomic_json_write, base_url_host_matches, base_url_hostname from hermes_constants import OPENROUTER_MODELS_URL logger = logging.getLogger(__name__) +# ``requests`` (with urllib3) costs ~27 ms of the `import cli` waterfall and +# is only used inside the fetch functions below. It's resolved lazily: +# ``_ensure_requests()`` populates the module global on the runtime path, and +# the PEP 562 ``__getattr__`` covers external attribute access — notably +# ``patch("agent.model_metadata.requests.get")`` in tests, which resolves the +# attribute at patch time. + + +def _ensure_requests(): + if "requests" not in globals(): + import requests as _requests + globals()["requests"] = _requests + return globals()["requests"] + + +def __getattr__(name: str): + if name == "requests": + return _ensure_requests() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + def _resolve_requests_verify() -> bool | str: """Resolve SSL verify setting for `requests` calls from env vars. @@ -50,7 +72,7 @@ def _resolve_requests_verify() -> bool | str: _PROVIDER_PREFIXES: frozenset[str] = frozenset({ "openrouter", "nous", "openai-codex", "copilot", "copilot-acp", "gemini", "ollama-cloud", "zai", "kimi-coding", "kimi-coding-cn", "stepfun", "minimax", "minimax-oauth", "minimax-cn", "anthropic", "deepseek", "deepinfra", - "opencode-zen", "opencode-go", "kilocode", "alibaba", "novita", + "opencode-zen", "opencode-go", "ai-gateway", "kilocode", "alibaba", "novita", "qwen-oauth", "xiaomi", "arcee", @@ -62,7 +84,7 @@ def _resolve_requests_verify() -> bool | str: "glm", "z-ai", "z.ai", "zhipu", "github", "github-copilot", "github-models", "kimi", "moonshot", "kimi-cn", "moonshot-cn", "claude", "deep-seek", "deep-infra", "ollama", - "stepfun", "opencode", "zen", "go", "kilo", "dashscope", "aliyun", "qwen", + "stepfun", "opencode", "zen", "go", "vercel", "kilo", "dashscope", "aliyun", "qwen", "mimo", "xiaomi-mimo", "tencent", "tokenhub", "tencent-cloud", "tencentmaas", "arcee-ai", "arceeai", @@ -123,6 +145,157 @@ def _strip_provider_prefix(model: str) -> str: _ENDPOINT_PROBE_TTL_SECONDS = 3600.0 _endpoint_probe_path_cache: Dict[str, tuple] = {} +# A configured endpoint that is routable-but-dead — e.g. a corp LAN address +# while off-VPN — blackholes TCP: the SYN draws no SYN-ACK, no RST and no ICMP +# error, so a probe waits out its full timeout instead of failing fast. Startup +# runs a whole waterfall of such probes across several functions here, and the +# stalls stack into a minute-long hang before the banner renders. +# +# Once ANY probe has actually observed a connect timeout for an endpoint, the +# others have nothing to gain by repeating it. Recording that observation and +# short-circuiting on it performs no network I/O of its own — it adds no probe +# for callers or tests to mock, and it can only ever fire after a real timeout +# has already been paid, so it cannot suppress a probe that would have worked. +_ENDPOINT_BLACKHOLE_TTL_SECONDS = 30.0 +# Values are monotonic timestamps of the last observed connect timeout. +_endpoint_blackhole_cache: Dict[str, float] = {} + + +def _endpoint_host_key(base_url: str) -> Optional[str]: + """Return a ``host:port`` key for ``base_url``, or None if it has no host. + + Keyed on host:port rather than the full URL so every probe path for one + server — ``/v1``-suffixed or not, LM Studio root or API root — shares a + single entry. + """ + normalized = _normalize_base_url(base_url) + if not normalized: + return None + url = normalized if "://" in normalized else f"http://{normalized}" + try: + parsed = urlparse(url) + host = parsed.hostname + port = parsed.port or (443 if parsed.scheme == "https" else 80) + except Exception: + return None + return f"{host}:{port}" if host else None + + +def _note_endpoint_blackholed(base_url: str) -> None: + """Record that a probe to ``base_url`` timed out during TCP connect.""" + key = _endpoint_host_key(base_url) + if key is None: + return + _endpoint_blackhole_cache[key] = time.monotonic() + logger.debug( + "Endpoint %s timed out connecting — skipping further probes for %.0fs", + key, _ENDPOINT_BLACKHOLE_TTL_SECONDS, + ) + + +def _endpoint_blackholed(base_url: str) -> bool: + """True if a recent probe to ``base_url`` timed out during TCP connect. + + Pure cache lookup; never touches the network. The entry expires after + _ENDPOINT_BLACKHOLE_TTL_SECONDS — long enough to collapse one startup's + burst of probes, short enough that bringing the VPN up mid-session is + picked up without a restart. + """ + if _ENDPOINT_BLACKHOLE_TTL_SECONDS <= 0: + return False + key = _endpoint_host_key(base_url) + if key is None: + return False + seen = _endpoint_blackhole_cache.get(key) + if seen is None: + return False + if (time.monotonic() - seen) >= _ENDPOINT_BLACKHOLE_TTL_SECONDS: + del _endpoint_blackhole_cache[key] + return False + return True + + +def _is_connect_timeout(exc: BaseException) -> bool: + """True for connect-phase timeouts raised by httpx or requests. + + Read timeouts are deliberately excluded: those mean the server accepted + the connection, which is the opposite of the blackhole this guards. + """ + try: + import httpx + if isinstance(exc, httpx.ConnectTimeout): + return True + except Exception: + pass + try: + from requests.exceptions import ConnectTimeout + if isinstance(exc, ConnectTimeout): + return True + except Exception: + pass + return False + +# ── Disk L2 for local-endpoint probe results ──────────────────────────────── +# The in-process caches above die with the process, so every CLI cold start +# with a local model re-paid the probe waterfall in AIAgent.__init__: +# detect_local_server_type (up to 4 HTTP GETs, ≤2 s each on a hung server) +# + /api/show (≤3 s). A short-TTL disk cache makes back-to-back CLI +# invocations hit disk instead of the network. Only SUCCESSFUL probes are +# persisted (a down server must not pin a negative verdict), and the TTL is +# short enough that swapping the server on a port (stop Ollama, start +# LM Studio) is picked up within minutes — strictly fresher than the 1 h +# in-process TTL that already accepts that staleness. +_LOCAL_PROBE_DISK_TTL_SECONDS = 300.0 + + +def _local_probe_disk_cache_path() -> Path: + from hermes_constants import get_hermes_home + return get_hermes_home() / "cache" / "local_endpoint_probes.json" + + +def _load_local_probe_disk_cache() -> Dict[str, Any]: + try: + with _local_probe_disk_cache_path().open("r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except Exception: + return {} + + +def _local_probe_disk_get(kind: str, key: str) -> Optional[Any]: + """Return a fresh cached value for ``kind:key``, else None.""" + entry = _load_local_probe_disk_cache().get(f"{kind}:{key}") + if not isinstance(entry, dict): + return None + try: + if (time.time() - float(entry["ts"])) >= _LOCAL_PROBE_DISK_TTL_SECONDS: + return None + return entry["value"] + except Exception: + return None + + +def _local_probe_disk_put(kind: str, key: str, value: Any) -> None: + """Persist a successful probe result. Best-effort; prunes stale entries.""" + try: + now = time.time() + data = _load_local_probe_disk_cache() + data = { + k: v + for k, v in data.items() + if isinstance(v, dict) + and (now - float(v.get("ts", 0))) < _LOCAL_PROBE_DISK_TTL_SECONDS + } + data[f"{kind}:{key}"] = {"value": value, "ts": now} + atomic_json_write( + _local_probe_disk_cache_path(), + data, + indent=0, + separators=(",", ":"), + ) + except Exception as e: + logger.debug("Failed to save local probe disk cache: %s", e) + def _get_model_metadata_cache_path() -> Path: """Return path to the OpenRouter model metadata disk cache.""" @@ -190,6 +363,27 @@ def _save_model_metadata_disk_cache(data: Dict[str, Dict[str, Any]]) -> None: # Default context length when no detection method succeeds. DEFAULT_FALLBACK_CONTEXT = CONTEXT_PROBE_TIERS[0] +# (model, base_url) pairs that already emitted the fallback warning. +# The fallback result itself is deliberately never cached, so without this +# the warning would repeat on every resolution for the same unknown model. +_FALLBACK_WARNED: set = set() + + +def _warn_context_length_fallback(model: str, base_url: str) -> None: + """Warn (once per model+endpoint) that context detection failed and the + hard default is being used, so small-context models (8K, 32K) don't + silently get 256K and cause hard-to-debug API failures.""" + key = (model, base_url or "") + if key in _FALLBACK_WARNED: + return + _FALLBACK_WARNED.add(key) + logger.warning( + "Could not determine context length for model %r (base_url=%s) " + "— falling back to %s tokens. Set model.context_length in " + "config.yaml to override.", + model, base_url or "default", f"{DEFAULT_FALLBACK_CONTEXT:,}", + ) + # Minimum context length required to run Hermes Agent. Models with fewer # tokens cannot maintain enough working memory for tool-calling workflows. # Sessions, model switches, and cron jobs should reject models below this. @@ -215,6 +409,7 @@ def _save_model_metadata_disk_cache(data: Dict[str, Dict[str, Any]]) -> None: # OpenRouter-prefixed models resolve via OpenRouter live API or models.dev. "claude-fable-5": 1000000, "claude-fable": 1000000, + "claude-opus-5": 1000000, "claude-sonnet-5": 1000000, "claude-opus-4-8": 1000000, "claude-opus-4.8": 1000000, @@ -550,12 +745,17 @@ def _is_known_provider_base_url(base_url: str) -> bool: def _endpoint_scoped_context_length(model: str, base_url: str) -> Optional[int]: - """Return metadata confirmed only for the Kimi Coding endpoint. + """Return context metadata confirmed for one provider endpoint. Kimi Coding serves K3 under the bare slug ``k3``, but users may also configure or select the public-facing aliases ``kimi-k3`` and ``kimi-k3-cot``. Only canonical ``https://api.kimi.com/coding`` endpoints (legacy Moonshot keys do not serve K3) get the 1 Mi context window. + + NVIDIA NIM serves ``deepseek-ai/deepseek-v4-pro`` with a 262,144-token + window even though DeepSeek's native endpoint serves the V4 family with a + 1M window. Keep the lower limit scoped to NVIDIA instead of weakening the + global model-family metadata. """ normalized = _normalize_base_url(base_url) try: @@ -575,6 +775,18 @@ def _endpoint_scoped_context_length(model: str, base_url: str) -> Optional[int]: and model.strip().lower() in {"k3", "kimi-k3", "kimi-k3-cot"} ): return 1_048_576 + if ( + parsed.scheme.lower() == "https" + and (parsed.hostname or "").lower() == "integrate.api.nvidia.com" + and port in (None, 443) + and parsed.username is None + and parsed.password is None + and parsed.path.rstrip("/") == "/v1" + and not parsed.query + and not parsed.fragment + and model.strip().lower() == "deepseek-ai/deepseek-v4-pro" + ): + return 262_144 return None @@ -714,7 +926,10 @@ def _localhost_to_ipv4(url: str) -> str: ``http://localhost...`` (e.g. ``?upstream=http://localhost:11434``) passes through untouched. """ - if not url: + if not url or not isinstance(url, str): + # Non-string values (test doubles, lazily-resolved config objects) + # previously flowed through these call sites untouched — keep that + # contract; re.sub would raise TypeError. return url return re.sub( r"^(https?://)localhost(?=[:/]|$)", @@ -751,8 +966,32 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]: if cached is not None and (time.monotonic() - cached[1]) < _ENDPOINT_PROBE_TTL_SECONDS: return cached[0] + # The host already blackholed a connect: skip the waterfall below, each leg + # of which would otherwise burn its full 2s timeout. Deliberately NOT + # written to _endpoint_probe_path_cache — that entry lives for an hour, + # which would pin the endpoint to "undetected" long after it comes back. + if _endpoint_blackholed(server_url): + return None + + # Disk L2: a fresh cross-process verdict skips the HTTP waterfall + # entirely (back-to-back CLI invocations, cron ticks). + disk_hit = _local_probe_disk_get("server_type", server_url) + if isinstance(disk_hit, str): + _endpoint_probe_path_cache[server_url] = (disk_hit, time.monotonic()) + return disk_hit + headers = _auth_headers(api_key) + def _probe_failed(exc: Exception) -> None: + """Swallow a probe error — or abort the waterfall if we were blackholed. + + Re-raising propagates out of the ``with`` block to the outer handler, + so the remaining legs are skipped instead of each stalling in turn. + """ + if _is_connect_timeout(exc): + _note_endpoint_blackholed(server_url) + raise exc + result: Optional[str] = None try: with httpx.Client(timeout=2.0, headers=headers) as client: @@ -761,8 +1000,8 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]: r = client.get(f"{lmstudio_url}/api/v1/models") if r.status_code == 200: result = "lm-studio" - except Exception: - pass + except Exception as exc: + _probe_failed(exc) if result is None: # Ollama exposes /api/tags and responds with {"models": [...]} # LM Studio returns {"error": "Unexpected endpoint"} with status 200 @@ -776,8 +1015,8 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]: result = "ollama" except Exception: pass - except Exception: - pass + except Exception as exc: + _probe_failed(exc) if result is None: # llama.cpp exposes /v1/props (older builds used /props without the /v1 prefix) try: @@ -786,8 +1025,8 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]: r = client.get(f"{server_url}/props") # fallback for older builds if r.status_code == 200 and "default_generation_settings" in r.text: result = "llamacpp" - except Exception: - pass + except Exception as exc: + _probe_failed(exc) if result is None: # vLLM: /version try: @@ -796,13 +1035,14 @@ def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]: data = r.json() if "version" in data: result = "vllm" - except Exception: - pass + except Exception as exc: + _probe_failed(exc) except Exception: pass if result is not None: _endpoint_probe_path_cache[server_url] = (result, time.monotonic()) + _local_probe_disk_put("server_type", server_url, result) return result @@ -925,6 +1165,7 @@ def fetch_model_metadata(force_refresh: bool = False) -> Dict[str, Dict[str, Any return _model_metadata_cache try: + _ensure_requests() # Tuple (connect, read) — flat timeout=10 means urllib3 can block 10s per # retry stage through proxies that 403 CONNECT, ballooning to minutes # (#46620). 5s connect / 10s read fails fast on unreachable hosts. @@ -953,7 +1194,7 @@ def fetch_model_metadata(force_refresh: bool = False) -> Dict[str, Dict[str, Any return cache except Exception as e: - logger.warning(f"Failed to fetch model metadata from OpenRouter: {e}") + logger.warning("Failed to fetch model metadata from OpenRouter: %s", e) if _model_metadata_cache: return _model_metadata_cache disk_cache = _load_model_metadata_disk_cache() @@ -981,6 +1222,7 @@ def fetch_endpoint_model_metadata( normalized = _normalize_base_url(base_url) if not normalized or _is_openrouter_base_url(normalized): return {} + _ensure_requests() if not force_refresh: cached = _endpoint_model_metadata_cache.get(normalized) @@ -988,6 +1230,12 @@ def fetch_endpoint_model_metadata( if cached is not None and (time.time() - cached_at) < _ENDPOINT_MODEL_CACHE_TTL: return cached + # Blackholed endpoint: every candidate below would spend its full 5s + # connect budget. Returned empty rather than cached, so the endpoint is + # retried as soon as the blackhole entry expires. + if _endpoint_blackholed(normalized): + return {} + candidates = [normalized] if normalized.endswith("/v1"): alternate = normalized[:-3].rstrip("/") @@ -1050,11 +1298,36 @@ def fetch_endpoint_model_metadata( return cache except Exception as exc: last_error = exc + if _is_connect_timeout(exc): + _note_endpoint_blackholed(normalized) for candidate in candidates: - url = candidate.rstrip("/") + "/models" + # A connect timeout on one candidate condemns the host, not the path: + # the remaining candidates differ only by URL suffix, so trying them + # would repeat the same stall. + if _endpoint_blackholed(normalized): + break + # normalized/candidates stay unrewritten (cache key stability); only + # the outbound request target is IPv4-resolved to skip the multi-second + # dual-stack IPv6 connect timeout (see _localhost_to_ipv4). + request_candidate = _localhost_to_ipv4(candidate) + url = request_candidate.rstrip("/") + "/models" + response = None try: - response = requests.get(url, headers=headers, timeout=(5, 10), verify=_resolve_requests_verify()) + response = requests.get( + url, + headers=headers, + timeout=(5, 10), + verify=_resolve_requests_verify(), + stream=True, + ) + if response.status_code in (401, 403): + logger.debug( + "Model metadata probe received HTTP %s from %s; stopping candidate probing", + response.status_code, + url, + ) + break response.raise_for_status() payload = response.json() cache: Dict[str, Dict[str, Any]] = {} @@ -1084,7 +1357,7 @@ def fetch_endpoint_model_metadata( if is_llamacpp: try: # Try /v1/props first (current llama.cpp); fall back to /props for older builds - base = candidate.rstrip("/").replace("/v1", "") + base = request_candidate.rstrip("/").replace("/v1", "") _verify = _resolve_requests_verify() props_resp = requests.get(base + "/v1/props", headers=headers, timeout=5, verify=_verify) if not props_resp.ok: @@ -1104,6 +1377,11 @@ def fetch_endpoint_model_metadata( return cache except Exception as exc: last_error = exc + if _is_connect_timeout(exc): + _note_endpoint_blackholed(normalized) + finally: + if response is not None: + response.close() if last_error: logger.debug("Failed to fetch model metadata from %s/models: %s", normalized, last_error) @@ -1522,6 +1800,13 @@ def query_ollama_num_ctx(model: str, base_url: str, api_key: str = "") -> Option if server_type != "ollama": return None + # Disk L2: /api/show results are stable for a given (model, server) on + # human timescales — skip the HTTP roundtrip on fresh cross-process hits. + _disk_key = f"{server_url}|{bare_model}" + disk_hit = _local_probe_disk_get("ollama_num_ctx", _disk_key) + if isinstance(disk_hit, int) and disk_hit > 0: + return disk_hit + headers = _auth_headers(api_key) try: @@ -1539,7 +1824,9 @@ def query_ollama_num_ctx(model: str, base_url: str, api_key: str = "") -> Option parts = line.strip().split() if len(parts) >= 2: try: - return int(parts[-1]) + _ctx = int(parts[-1]) + _local_probe_disk_put("ollama_num_ctx", _disk_key, _ctx) + return _ctx except ValueError: pass @@ -1547,7 +1834,9 @@ def query_ollama_num_ctx(model: str, base_url: str, api_key: str = "") -> Option model_info = data.get("model_info", {}) for key, value in model_info.items(): if "context_length" in key and isinstance(value, (int, float)): - return int(value) + _ctx = int(value) + _local_probe_disk_put("ollama_num_ctx", _disk_key, _ctx) + return _ctx except Exception: pass return None @@ -1653,6 +1942,9 @@ def _query_ollama_api_show_uncached(model: str, base_url: str, api_key: str = "" if server_url.endswith("/v1"): server_url = server_url[:-3] + if _endpoint_blackholed(server_url): + return None + headers = _auth_headers(api_key) try: @@ -1684,8 +1976,9 @@ def _query_ollama_api_show_uncached(model: str, base_url: str, api_key: str = "" return ctx except ValueError: pass - except Exception: - pass + except Exception as exc: + if _is_connect_timeout(exc): + _note_endpoint_blackholed(server_url) return None @@ -1773,6 +2066,9 @@ def _query_local_context_length_uncached(model: str, base_url: str, api_key: str server_url = server_url[:-3] lmstudio_url = _localhost_to_ipv4(_lmstudio_server_root(base_url)) + if _endpoint_blackholed(server_url): + return None + headers = _auth_headers(api_key) try: @@ -1848,8 +2144,9 @@ def _query_local_context_length_uncached(model: str, base_url: str, api_key: str ctx = m.get("max_model_len") or m.get("context_length") or m.get("max_tokens") if ctx and isinstance(ctx, (int, float)): return int(ctx) - except Exception: - pass + except Exception as exc: + if _is_connect_timeout(exc): + _note_endpoint_blackholed(server_url) return None @@ -1881,6 +2178,7 @@ def _query_anthropic_context_length(model: str, base_url: str, api_key: str) -> "x-api-key": api_key, "anthropic-version": "2023-06-01", } + _ensure_requests() resp = requests.get(url, headers=headers, timeout=(5, 10), verify=_resolve_requests_verify()) if resp.status_code != 200: return None @@ -1988,6 +2286,7 @@ def _fetch_codex_oauth_context_lengths_with_source( headers["ChatGPT-Account-Id"] = acct_id try: + _ensure_requests() resp = requests.get( "https://chatgpt.com/backend-api/codex/models?client_version=1.0.0", headers=headers, @@ -2428,21 +2727,27 @@ def get_model_context_length( if context_length is not None: return context_length if not _is_known_provider_base_url(base_url): - # 2b. Ollama native /api/show — any URL might be an Ollama server - # (local, cloud, or custom hosting). Non-Ollama servers return - # 404/405 quickly. Fall through on failure. - ctx = _query_ollama_api_show(model, base_url, api_key=api_key) - if ctx is not None: - if not _skip_persistent_context_cache(base_url, provider): - save_context_length(model, base_url, ctx) - return ctx - # 3. Try querying local server directly + # For local endpoints, run the probe that respects configured + # Modelfile context values first. _query_local_context_length + # prefers num_ctx from Modelfile, while _query_ollama_api_show + # returns the GGUF training max first which can be larger and + # would create a false-safe window for compression (#63122). + # Non-local endpoints preserve the existing GGUF-first behavior. if is_local_endpoint(base_url): local_ctx = _query_local_context_length(model, base_url, api_key=api_key) if local_ctx and local_ctx > 0: if not _skip_persistent_context_cache(base_url, provider): _maybe_cache_local_context_length(model, base_url, local_ctx) return local_ctx + # 2b. Ollama native /api/show — non-local endpoints preserve + # the existing generic /api/show GGUF-first behavior. + # Non-Ollama servers return 404/405 quickly. + ctx = _query_ollama_api_show(model, base_url, api_key=api_key) + if ctx is not None: + if not _skip_persistent_context_cache(base_url, provider): + save_context_length(model, base_url, ctx) + return ctx + # 3. Probe-down fallback after endpoint-specific detection failed logger.info( "Could not detect context length for model %r at %s — " "defaulting to %s tokens (probe-down). Set model.context_length " @@ -2469,6 +2774,9 @@ def get_model_context_length( f"{length:,}", model, default_model, ) return length + # Same silent-256K bug class as the step-9 fallback below — + # warn here too so custom/local endpoints aren't left invisible. + _warn_context_length_fallback(model, base_url) return DEFAULT_FALLBACK_CONTEXT # 4. Anthropic /v1/models API (only for regular API keys, not OAuth) @@ -2641,7 +2949,10 @@ def get_model_context_length( if default_model in model_lower: return length - # 9. Default fallback — 256K + # 9. Default fallback — warn (deduped per model+endpoint) so + # small-context models don't silently get 256K. See + # _warn_context_length_fallback for rationale. + _warn_context_length_fallback(model, base_url) return DEFAULT_FALLBACK_CONTEXT @@ -2738,14 +3049,92 @@ def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: image — the Anthropic pricing model — instead of counting raw base64 character length. Without this, a single ~1MB screenshot would be estimated at ~250K tokens and trigger premature context compression. + + Per-message results are memoized (see ``_estimate_message_tokens_cached``) + keyed on a deep *identity fingerprint* of the message, so re-walking a + long history every iteration only pays for messages whose object graph + actually changed. The memo is exact: equal fingerprints imply identical + leaf objects and structure, hence an identical estimate. """ _IMAGE_TOKEN_COST = 1500 - text_tokens = 0 - image_tokens = 0 + total = 0 for msg in messages: - text_tokens += _estimate_message_tokens_without_images(msg) - image_tokens += _count_image_tokens(msg, _IMAGE_TOKEN_COST) - return text_tokens + image_tokens + total += _estimate_message_tokens_cached(msg, _IMAGE_TOKEN_COST) + return total + + +# --- Per-message token-estimate memo ------------------------------------- +# +# ``estimate_messages_tokens_rough`` is called on the full history every +# loop iteration (conversation_loop preflight), repeatedly during compaction +# telemetry, and inside an O(n^2) shrink loop in moa_loop. The per-message +# helpers are pure functions of the message's value, so a memo keyed on a +# fingerprint that uniquely determines the value is exactly equivalent. +# +# Fingerprint design (soundness argument): +# * strings are fingerprinted by ``id()`` AND pinned (a strong reference is +# stored in the cache entry). While the entry lives, that id cannot be +# reused by another object, so id-equality implies object-equality — +# strings are immutable, so value-equality too (no #50372-style aliasing). +# * ints/floats/bools/None are fingerprinted by value. +# * dicts/lists recurse structurally, preserving key order — ``str(shadow)`` +# depends on insertion order, so order is part of the key. +# * any other type aborts the memo and falls through to a direct compute. +# Equal fingerprints therefore imply deep-equal messages built from identical +# immutable leaves ⇒ identical ``str(shadow)`` bytes ⇒ identical estimate. +# +# Because the api_messages build shallow-copies history dicts each iteration, +# the copies share the same content strings — so unchanged history messages +# hit the memo even though the outer dicts are fresh objects every turn. +_MSG_TOKENS_CACHE: Dict[Any, Tuple[list, int]] = {} +_MSG_TOKENS_CACHE_MAX = 4096 + + +def _msg_fingerprint(value: Any, pins: list) -> Any: + if value is None or value is True or value is False: + return value + t = type(value) + if t is str: + pins.append(value) + return ("s", id(value)) + if t is int or t is float: + return ("n", t.__name__, value) + if t is dict: + return ("d", tuple( + (_msg_fingerprint(k, pins), _msg_fingerprint(v, pins)) + for k, v in value.items() + )) + if t is list: + return ("l", tuple(_msg_fingerprint(v, pins) for v in value)) + if t is tuple: + return ("t", tuple(_msg_fingerprint(v, pins) for v in value)) + raise ValueError("unfingerprintable message value") + + +def _estimate_message_tokens_cached(msg: Any, image_cost: int) -> int: + try: + pins: list = [] + key = _msg_fingerprint(msg, pins) + hash(key) + except Exception: + return ( + _estimate_message_tokens_without_images(msg) + + _count_image_tokens(msg, image_cost) + ) + cached = _MSG_TOKENS_CACHE.get(key) + if cached is not None: + return cached[1] + tokens = ( + _estimate_message_tokens_without_images(msg) + + _count_image_tokens(msg, image_cost) + ) + _MSG_TOKENS_CACHE[key] = (pins, tokens) + while len(_MSG_TOKENS_CACHE) > _MSG_TOKENS_CACHE_MAX: + try: + _MSG_TOKENS_CACHE.pop(next(iter(_MSG_TOKENS_CACHE))) + except (StopIteration, KeyError, RuntimeError): + break + return tokens def _count_image_tokens(msg: Dict[str, Any], cost_per_image: int) -> int: @@ -2774,19 +3163,48 @@ def _count_image_tokens(msg: Dict[str, Any], cost_per_image: int) -> int: return count * cost_per_image -def _estimate_message_chars(msg: Dict[str, Any]) -> int: - """Char count for token estimation, excluding base64 image data. +def _wire_message_shadow(msg: Dict[str, Any]) -> Dict[str, Any]: + """Shadow of a message holding only what the provider actually receives. - Base64 images are counted via `_count_image_tokens` instead; including - their raw chars here would massively overestimate token usage. + Two adjustments to the raw persisted dict: + + * ``api_content`` is a SUBSTITUTE for ``content``, not an addition to it. + ``turn_context.substitute_api_content()`` pops the sidecar and overwrites + ``content`` at every API-bound build site, so exactly one of the two is + ever sent. Counting both double-counts any message whose sidecar differs + from its clean stored content (2.00x on a 40KB sidecar). + + The substitution mirrors that helper's guard exactly: only a non-empty + STRING sidecar on a ``user``/``assistant`` row displaces ``content``. + Any other sidecar shape is popped and discarded on the wire without + touching ``content``, so a shadow that substituted unconditionally + would UNDERcount those rows — the dangerous direction, since it makes + compaction fire too late and the turn dies on a hard context error. + * Base64 image payloads are replaced with a placeholder; they are charged + separately at a flat rate by ``_count_image_tokens``, and counting their + raw chars here would massively overestimate usage. """ - if not isinstance(msg, dict): - return len(str(msg)) + sidecar = msg.get("api_content") + sidecar_wins = ( + isinstance(sidecar, str) + and bool(sidecar) + and msg.get("role") in ("user", "assistant") + ) shadow: Dict[str, Any] = {} for k, v in msg.items(): - if k == "_anthropic_content_blocks": + if k in ("_anthropic_content_blocks", "reasoning_details"): + continue + if k == "api_content": + # Always popped before the request is built; only counted when it + # actually replaces ``content``. + if sidecar_wins: + shadow["content"] = v continue if k == "content": + if sidecar_wins: + # The sidecar wins on the wire; skip the clean copy so the + # same logical content is not counted twice. + continue if isinstance(v, list): cleaned = [] for part in v: @@ -2804,36 +3222,25 @@ def _estimate_message_chars(msg: Dict[str, Any]) -> int: shadow[k] = v else: shadow[k] = v - return len(str(shadow)) + return shadow + + +def _estimate_message_chars(msg: Dict[str, Any]) -> int: + """Char count for token estimation, excluding base64 image data. + + Base64 images are counted via `_count_image_tokens` instead; including + their raw chars here would massively overestimate token usage. + """ + if not isinstance(msg, dict): + return len(str(msg)) + return len(str(_wire_message_shadow(msg))) def _estimate_message_tokens_without_images(msg: Dict[str, Any]) -> int: """Token estimate for a message shadow with image payloads stripped.""" if not isinstance(msg, dict): return estimate_tokens_rough(str(msg)) - shadow: Dict[str, Any] = {} - for k, v in msg.items(): - if k == "_anthropic_content_blocks": - continue - if k == "content": - if isinstance(v, list): - cleaned = [] - for part in v: - if isinstance(part, dict): - if part.get("type") in {"image", "image_url", "input_image"}: - cleaned.append({"type": part.get("type"), "image": "[stripped]"}) - else: - cleaned.append(part) - else: - cleaned.append(part) - shadow[k] = cleaned - elif isinstance(v, dict) and v.get("_multimodal"): - shadow[k] = v.get("text_summary", "") - else: - shadow[k] = v - else: - shadow[k] = v - return estimate_tokens_rough(str(shadow)) + return estimate_tokens_rough(str(_wire_message_shadow(msg))) def estimate_request_tokens_rough( diff --git a/agent/models_dev.py b/agent/models_dev.py index 590f77806abf..71af78625a42 100644 --- a/agent/models_dev.py +++ b/agent/models_dev.py @@ -8,11 +8,15 @@ (reasoning, tools, vision, PDF, audio), modalities, knowledge cutoff, open-weights flag, family grouping, deprecation status -Data resolution order (like TypeScript OpenCode): - 1. Bundled snapshot (ships with the package — offline-first) - 2. Disk cache (~/.hermes/models_dev_cache.json) - 3. Network fetch (https://models.dev/api.json) - 4. Background refresh every 60 minutes +Data resolution order: + 1. In-memory cache (fresh, or stale served immediately while a single + background daemon thread refreshes) + 2. Disk cache (~/.hermes/models_dev_cache.json — any age; stale data is + served rather than blocking callers on the network) + 3. Network fetch (https://models.dev/api.json) — only when no cache + exists at all; failed refreshes back off for 5 minutes process-wide +Latency-sensitive callers (gateway route-identity checks) pass +``allow_network=False`` and never touch the network. Other modules should import the dataclasses and query functions from here rather than parsing the raw JSON themselves. @@ -20,6 +24,7 @@ import json import logging +import threading import time from dataclasses import dataclass from pathlib import Path @@ -33,10 +38,15 @@ MODELS_DEV_URL = "https://models.dev/api.json" _MODELS_DEV_CACHE_TTL = 3600 # 1 hour in-memory +_MODELS_DEV_RETRY_DELAY = 300 # 5 minutes after a failed refresh # In-memory cache _models_dev_cache: Dict[str, Any] = {} _models_dev_cache_time: float = 0 +_models_dev_retry_after: float = 0 +_models_dev_fetch_lock = threading.Lock() +_models_dev_refresh_lock = threading.Lock() +_models_dev_refresh_in_flight = False # --------------------------------------------------------------------------- @@ -158,6 +168,7 @@ class ProviderInfo: "alibaba": "alibaba", "qwen-oauth": "alibaba", "copilot": "github-copilot", + "ai-gateway": "vercel", "opencode-zen": "opencode", "opencode-go": "opencode-go", "kilocode": "kilo", @@ -237,27 +248,157 @@ def _save_disk_cache(data: Dict[str, Any]) -> None: logger.debug("Failed to save models.dev disk cache: %s", e) -def fetch_models_dev(force_refresh: bool = False) -> Dict[str, Any]: +def _fetch_models_dev_from_network() -> Dict[str, Any]: + """Fetch the live models.dev registry without touching local caches. + + Raises on network errors and on an empty/invalid registry payload. + """ + # Tuple (connect, read): a flat timeout=15 let a blackholed connect + # stall the first-turn critical path for the full 15 s. 5 s connect + # fails fast on unreachable hosts; 10 s read still tolerates a slow + # registry response (matches the OpenRouter fetch convention in + # agent/model_metadata.py). + response = requests.get(MODELS_DEV_URL, timeout=(5, 10)) + response.raise_for_status() + data = response.json() + if not isinstance(data, dict) or not data: + raise ValueError("models.dev returned an empty or invalid registry") + return data + + +def _mark_stale_cache_grace() -> None: + """Give stale cache data a short in-memory grace before retrying refresh. + + Only ever moves the timestamp forward: if a background refresh completed + between the caller's staleness check and this call, the fresh timestamp + is preserved instead of being rewound to a 5-minute grace. + """ + global _models_dev_cache_time + grace_time = time.time() - _MODELS_DEV_CACHE_TTL + _MODELS_DEV_RETRY_DELAY + if grace_time > _models_dev_cache_time: + _models_dev_cache_time = grace_time + + +def _commit_registry(data: Dict[str, Any], *, where: str) -> None: + """Persist a freshly fetched registry: disk + in-mem + clear backoff. + + Callers must hold ``_models_dev_fetch_lock`` so a failing refresh on one + path can never stomp the state a succeeding refresh on the other path + just committed (e.g. a failing background worker re-arming the backoff + immediately after a successful ``force_refresh``). + """ + global _models_dev_cache, _models_dev_cache_time, _models_dev_retry_after + _save_disk_cache(data) + _models_dev_cache = data + _models_dev_cache_time = time.time() + _models_dev_retry_after = 0 + logger.debug( + "Refreshed models.dev registry (%s): %d providers, %d total models", + where, + len(data), + sum(len(p.get("models", {})) for p in data.values() if isinstance(p, dict)), + ) + + +def _note_refresh_failure(exc: Exception, *, where: str) -> None: + """Record a failed refresh: arm the process-wide 5-minute backoff. + + Callers must hold ``_models_dev_fetch_lock`` (see ``_commit_registry``). + """ + global _models_dev_retry_after + _models_dev_retry_after = time.time() + _MODELS_DEV_RETRY_DELAY + logger.debug( + "models.dev refresh failed (%s); retry suppressed for %ds: %s", + where, + _MODELS_DEV_RETRY_DELAY, + exc, + ) + + +def _background_refresh_models_dev() -> None: + """Best-effort refresh after serving stale cache data.""" + global _models_dev_refresh_in_flight + try: + data = _fetch_models_dev_from_network() + with _models_dev_fetch_lock: + _commit_registry(data, where="background") + except Exception as e: + with _models_dev_fetch_lock: + _note_refresh_failure(e, where="background") + finally: + with _models_dev_refresh_lock: + _models_dev_refresh_in_flight = False + + +def _start_background_refresh_models_dev() -> None: + """Start one daemon refresh worker if none is already running. + + Honors the process-wide failure backoff: after a failed refresh, + no new background worker is spawned until ``_models_dev_retry_after``. + """ + global _models_dev_refresh_in_flight + if time.time() < _models_dev_retry_after: + return + with _models_dev_refresh_lock: + if _models_dev_refresh_in_flight: + return + _models_dev_refresh_in_flight = True + thread = threading.Thread( + target=_background_refresh_models_dev, + name="models-dev-refresh", + daemon=True, + ) + try: + thread.start() + except Exception as e: + # Thread/fd exhaustion: clear the flag so refresh isn't disabled + # for the rest of the process lifetime. Callers still get stale data. + with _models_dev_refresh_lock: + _models_dev_refresh_in_flight = False + logger.debug("Failed to start models.dev refresh thread: %s", e) + + +def fetch_models_dev( + force_refresh: bool = False, *, allow_network: bool = True +) -> Dict[str, Any]: """Fetch models.dev registry. Cache hierarchy: in-mem → disk → network. Returns the full registry dict keyed by provider ID, or empty dict on failure. Cache hierarchy (when ``force_refresh=False``): - 1. In-memory cache, populated and < TTL old → return immediately. - 2. **Disk cache file < TTL old by mtime → load, populate in-mem, return.** - No network call. Saves ~500 ms per cold-start agent construction; - ``models.dev`` only changes when providers add new models, so a - 1 hour staleness window is acceptable (same TTL as in-mem cache). - 3. Network fetch → on success, save to disk + in-mem and return. - 4. Network fails → fall back to ANY available disk cache (even stale) - with a short 5 min in-mem grace period before retrying network. + 1. Fresh in-memory cache → return immediately. + 2. Stale in-memory cache → return immediately and refresh in a single + background daemon thread. Callers never block on the network while + any cache exists; ``models.dev`` only changes when providers add + new models, so stale data is preferable to a foreground timeout. + 3. Disk cache file (any age) → load, populate in-mem, return + immediately. Stale disk caches trigger the same background refresh. + 4. No cache at all → singleflight foreground network fetch. On + success, save to disk + in-mem and return. + 5. Any failed refresh (foreground or background) suppresses further + automatic refreshes for 5 minutes process-wide. When ``force_refresh=True`` (used by ``hermes config refresh``, the - \"refresh model catalog\" code path), stages 1 and 2 are skipped. The - function always hits the network and only falls back to disk if the - network call fails. + \"refresh model catalog\" code path), cache fast paths and the failure + backoff are bypassed; the function hits the network and only falls back + to cached data if the call fails. When ``allow_network=False``, any + memory or disk cache is returned regardless of age and no request is + made — used by latency-sensitive paths (gateway route-identity checks) + that must never wait on the network. """ - global _models_dev_cache, _models_dev_cache_time + global _models_dev_cache, _models_dev_cache_time, _models_dev_retry_after + + if not allow_network: + if _models_dev_cache: + return _models_dev_cache + disk_data = _load_disk_cache() + if disk_data: + _models_dev_cache = disk_data + disk_age = _disk_cache_age_seconds() + _models_dev_cache_time = ( + time.time() - disk_age if disk_age is not None else 0 + ) + return _models_dev_cache # Stage 1: fresh in-memory cache wins. This is the hot path on # long-lived processes — no I/O, no system calls. @@ -268,54 +409,82 @@ def fetch_models_dev(force_refresh: bool = False) -> Dict[str, Any]: ): return _models_dev_cache - # Stage 2: fresh-by-mtime disk cache short-circuits the network call. - # Only kicks in on cold-start processes (in-mem cache is empty or - # expired) and only when the user hasn't asked for a forced refresh. - # Skipped if the disk cache file is missing, unreadable, or older - # than _MODELS_DEV_CACHE_TTL. + # Stage 2: stale in-memory cache is still better than blocking provider + # resolution on a foreground network timeout. Refresh it in the background. + if not force_refresh and _models_dev_cache: + _mark_stale_cache_grace() + _start_background_refresh_models_dev() + logger.debug( + "Using stale in-memory models.dev cache; refreshing in background" + ) + return _models_dev_cache + + # Stage 3: disk cache short-circuits the network call. + # Only kicks in on cold-start processes (in-mem cache is empty) and only + # when the user hasn't asked for a forced refresh. A stale disk cache is + # deliberately usable: provider/model resolution should not hang just + # because models.dev is unreachable. if not force_refresh: disk_age = _disk_cache_age_seconds() - if disk_age is not None and disk_age < _MODELS_DEV_CACHE_TTL: + if disk_age is not None: disk_data = _load_disk_cache() if disk_data: _models_dev_cache = disk_data - # Anchor in-mem TTL to the disk file's age so we don't - # extend an already-aging cache by another full hour. - _models_dev_cache_time = time.time() - disk_age - logger.debug( - "Loaded models.dev from fresh disk cache " - "(%d providers, age=%.0fs)", len(disk_data), disk_age, - ) + if disk_age < _MODELS_DEV_CACHE_TTL: + # Anchor in-mem TTL to the disk file's age so we don't + # extend an already-aging cache by another full hour. + _models_dev_cache_time = time.time() - disk_age + logger.debug( + "Loaded models.dev from fresh disk cache " + "(%d providers, age=%.0fs)", len(disk_data), disk_age, + ) + else: + _mark_stale_cache_grace() + _start_background_refresh_models_dev() + logger.debug( + "Using stale models.dev disk cache (age=%.0fs); " + "refreshing in background", + disk_age, + ) return _models_dev_cache - # Stage 3: network fetch. - try: - response = requests.get(MODELS_DEV_URL, timeout=15) - response.raise_for_status() - data = response.json() - if isinstance(data, dict) and data: - _models_dev_cache = data - _models_dev_cache_time = time.time() - _save_disk_cache(data) - logger.debug( - "Fetched models.dev registry: %d providers, %d total models", - len(data), - sum(len(p.get("models", {})) for p in data.values() if isinstance(p, dict)), - ) - return data - except Exception as e: - logger.debug("Failed to fetch models.dev: %s", e) + # Failed automatic refreshes are process-wide. Avoid making every caller + # retry the same unreachable endpoint while no usable cache exists. + if not force_refresh and time.time() < _models_dev_retry_after: + return _models_dev_cache - # Stage 4: network failed — fall back to whatever disk cache exists, - # even if it's stale. Give it a short 5 min in-mem TTL so we retry - # the network soon instead of serving stale data for a full hour. - if not _models_dev_cache: - _models_dev_cache = _load_disk_cache() - if _models_dev_cache: - _models_dev_cache_time = time.time() - _MODELS_DEV_CACHE_TTL + 300 - logger.debug("Loaded models.dev from disk cache (%d providers)", len(_models_dev_cache)) + # Stage 4: singleflight foreground network fetch — only reached when no + # memory or disk cache exists (or on force_refresh). Recheck state after + # acquiring the lock because another caller may have refreshed or + # established backoff while we waited. + with _models_dev_fetch_lock: + now = time.time() + if not force_refresh: + if _models_dev_cache: + return _models_dev_cache + if now < _models_dev_retry_after: + return _models_dev_cache + + try: + data = _fetch_models_dev_from_network() + _commit_registry(data, where="foreground") + return data + except Exception as e: + _note_refresh_failure(e, where="foreground") + + # Stage 5: network failed — return any stale memory/disk cache. Cache + # freshness remains expired; the retry-after timestamp controls when + # the next automatic request is allowed. + if not _models_dev_cache: + _models_dev_cache = _load_disk_cache() + _models_dev_cache_time = 0 + if _models_dev_cache: + logger.debug( + "Loaded stale models.dev disk cache (%d providers)", + len(_models_dev_cache), + ) - return _models_dev_cache + return _models_dev_cache def lookup_models_dev_context(provider: str, model: str) -> Optional[int]: @@ -671,7 +840,9 @@ def _parse_provider_info(provider_id: str, raw: Dict[str, Any]) -> ProviderInfo: # Provider-level queries # --------------------------------------------------------------------------- -def get_provider_info(provider_id: str) -> Optional[ProviderInfo]: +def get_provider_info( + provider_id: str, *, allow_network: bool = True +) -> Optional[ProviderInfo]: """Get full provider metadata from models.dev. Accepts either a Hermes provider ID (e.g. "kilocode") or a models.dev @@ -680,7 +851,14 @@ def get_provider_info(provider_id: str) -> Optional[ProviderInfo]: # Resolve Hermes ID → models.dev ID mdev_id = PROVIDER_TO_MODELS_DEV.get(provider_id, provider_id) - data = fetch_models_dev() + # NOTE: keep the zero-argument call on the default path. Dozens of test + # sites monkeypatch fetch_models_dev with zero-arg lambdas; passing the + # kwarg unconditionally would break them all (they raise TypeError). + data = ( + fetch_models_dev() + if allow_network + else fetch_models_dev(allow_network=False) + ) raw = data.get(mdev_id) if not isinstance(raw, dict): return None diff --git a/agent/monitoring/__init__.py b/agent/monitoring/__init__.py new file mode 100644 index 000000000000..ee45ff01649e --- /dev/null +++ b/agent/monitoring/__init__.py @@ -0,0 +1,29 @@ +"""Hermes gateway monitoring. + +Service health monitoring plus redacted operational diagnostics for the +gateway daemon, exported over OTLP to an operator-configured endpoint. + +``emitter`` is the in-process event bus: producers (gateway status hooks, +the diagnostic log handler) hand typed events to a fire-and-forget queue, +and subscribers (the OTLP streamers) consume them off the hot path. The +emitter never blocks or raises into gateway code (the hot-path invariant), +and nothing is persisted locally — monitoring is an egress path, not a store. + +Deliberately out of scope here: run/model/tool trajectory capture, usage +analytics, and any content-bearing signal. Those planes are served by the +NeMo Relay integration and its Hermes-owned subscribers. +""" + +from __future__ import annotations + +from . import emitter, events + +emit = emitter.emit +get_emitter = emitter.get_emitter + +__all__ = [ + "emitter", + "events", + "emit", + "get_emitter", +] diff --git a/agent/monitoring/cron_health.py b/agent/monitoring/cron_health.py new file mode 100644 index 000000000000..2ed0f1cb9b2e --- /dev/null +++ b/agent/monitoring/cron_health.py @@ -0,0 +1,201 @@ +"""Content-free cron service-health and execution telemetry projection.""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from agent.monitoring.events import CronExecutionEvent +from agent.monitoring.gateway_health import GatewayHealthSnapshot, GatewayMetric +from cron.jobs import ( + _compute_grace_seconds, + get_catch_up_occurrence_count, + get_ticker_heartbeat_age, + get_ticker_success_age, + load_jobs, +) +from cron.scheduler import get_running_job_ids +from hermes_time import now as _hermes_now + +logger = logging.getLogger(__name__) +_KNOWN_STATUSES = {"claimed", "running", "completed", "failed", "unknown"} +_KNOWN_SOURCES = {"builtin", "direct", "external"} +_KNOWN_DELIVERY_OUTCOMES = {"delivered", "failed", "suppressed", "not_configured"} + + +@dataclass(frozen=True, slots=True) +class CronHealthSnapshot: + metrics: list[GatewayMetric] + events: list[CronExecutionEvent] + + +def _now() -> datetime: + return _hermes_now() + + +def _job_key(raw: Any) -> str: + value = str(raw or "unknown").encode("utf-8", errors="replace") + return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" + + +def classify_cron_error(raw: Any) -> str: + text = str(raw or "").lower() + if ( + re.search(r"\b(?:authentication|authenticated|authenticate|authorization|authorized|authorize|unauthorized|forbidden)\b", text) + or re.search(r"\bbearer\b", text) + or re.search(r"\b(?:access|api|refresh) token\b", text) + or re.search(r"\b(?:401|403)\b", text) + ): + return "auth_failed" + if "rate limit" in text or "429" in text or "quota" in text: + return "rate_limited" + if "timeout" in text or "timed out" in text: + return "timeout" + if any(value in text for value in ("network", "connection", "dns", "socket", "unreachable")): + return "network_error" + if "dispatch" in text or "executor" in text: + return "dispatch_failed" + if "interrupt" in text or "owner exited" in text or "restarted" in text: + return "interrupted" + if "empty response" in text: + return "empty_response" + if any(value in text for value in ("config", "missing", "invalid")): + return "invalid_config" + return "unknown" + + +def _parse_time(raw: Any) -> Optional[datetime]: + try: + return datetime.fromisoformat(str(raw)) if raw else None + except (TypeError, ValueError): + return None + + +def _duration_ms(record: dict[str, Any]) -> Optional[int]: + start = _parse_time(record.get("started_at")) or _parse_time(record.get("claimed_at")) + finish = _parse_time(record.get("finished_at")) + if start is None or finish is None: + return None + try: + duration = int((finish - start).total_seconds() * 1000) + except (TypeError, ValueError): + return None + return max(0, duration) + + +def project_execution_event( + record: dict[str, Any], *, delivery_outcome: Optional[str] = None +) -> CronExecutionEvent: + status = str(record.get("status") or "unknown").lower() + source = str(record.get("source") or "unknown").lower() + if source not in _KNOWN_SOURCES and source != "unknown": + source = "external" + outcome = str(delivery_outcome).lower() if delivery_outcome is not None else None + return CronExecutionEvent( + status=status if status in _KNOWN_STATUSES else "unknown", + job_key=_job_key(record.get("job_id")), + source=source if source in _KNOWN_SOURCES else "unknown", + duration_ms=_duration_ms(record), + delivery_outcome=( + outcome if outcome in _KNOWN_DELIVERY_OUTCOMES else None + ), + error_class=( + classify_cron_error(record.get("error")) + if status in {"failed", "unknown"} + else None + ), + ) + + +def emit_execution_state( + record: Optional[dict[str, Any]], *, delivery_outcome: Optional[str] = None +) -> None: + """Best-effort lifecycle emit; terminal states synchronously cross the queue barrier.""" + if not record: + return + try: + from agent.monitoring import emitter + + event = project_execution_event(record, delivery_outcome=delivery_outcome) + target = emitter.get_emitter() + target.emit(event) + if event.status in {"completed", "failed", "unknown"}: + target.flush(timeout=1.0) + except Exception: + logger.debug("cron execution telemetry emit failed", exc_info=True) + + +def _is_overdue(job: dict[str, Any], now: datetime) -> bool: + if not job.get("enabled", True): + return False + next_run = _parse_time(job.get("next_run_at")) + schedule = job.get("schedule") + if next_run is None or not isinstance(schedule, dict): + return False + try: + if next_run.tzinfo is None and now.tzinfo is not None: + next_run = next_run.replace(tzinfo=now.tzinfo) + lateness = (now - next_run).total_seconds() + return lateness > _compute_grace_seconds(schedule) + except (TypeError, ValueError): + return False + + +def build_cron_health_snapshot() -> CronHealthSnapshot: + metrics: list[GatewayMetric] = [] + for name, reader in ( + ("hermes.cron.scheduler.heartbeat_age_seconds", get_ticker_heartbeat_age), + ("hermes.cron.scheduler.last_success_age_seconds", get_ticker_success_age), + ): + try: + value = reader() + if value is not None: + metrics.append(GatewayMetric(name, max(0.0, float(value)), {})) + except Exception: + logger.debug("cron freshness metric unavailable", exc_info=True) + + try: + metrics.append( + GatewayMetric( + "hermes.cron.scheduler.catch_up_occurrences", + get_catch_up_occurrence_count(), + {}, + ) + ) + except Exception: + logger.debug("cron catch-up metric unavailable", exc_info=True) + + try: + jobs = load_jobs() + enabled = [job for job in jobs if job.get("enabled", True)] + metrics.append(GatewayMetric("hermes.cron.jobs.enabled", len(enabled), {})) + metrics.append( + GatewayMetric( + "hermes.cron.jobs.overdue", + sum(1 for job in enabled if _is_overdue(job, _now())), + {}, + ) + ) + except Exception: + logger.debug("cron job metrics unavailable", exc_info=True) + + try: + metrics.append( + GatewayMetric("hermes.cron.jobs.running", len(get_running_job_ids()), {}) + ) + except Exception: + logger.debug("cron running-job metric unavailable", exc_info=True) + return CronHealthSnapshot(metrics=metrics, events=[]) + + +__all__ = [ + "CronHealthSnapshot", + "build_cron_health_snapshot", + "classify_cron_error", + "emit_execution_state", + "project_execution_event", +] diff --git a/agent/monitoring/emitter.py b/agent/monitoring/emitter.py new file mode 100644 index 000000000000..18a0d4851ae0 --- /dev/null +++ b/agent/monitoring/emitter.py @@ -0,0 +1,211 @@ +"""Monitoring emitter: fire-and-forget queue + background dispatcher. + +The emitter is the single seam between producers (gateway status hooks, the +diagnostic log handler) and consumers (the OTLP streamers). Its contract is +the hot-path invariant: + + ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, + and MUST NEVER raise into the caller. A monitoring failure is logged + locally and dropped — it can never affect the gateway or a session. + +Mechanism: + * ``emit(event)`` does a non-blocking ``queue.put_nowait`` wrapped in a bare + except. On a full queue it drops the *oldest* event and counts the drop. + * A daemon thread drains the queue and fans each batch out to subscribers + (the OTLP metric/span/log streamers). Each subscriber is fail-isolated — + a slow or raising subscriber never affects the hot path or its peers. + +Nothing is persisted here. Monitoring is an egress path, not a local store; +if no subscriber is attached, events simply age out of the ring buffer. +""" + +from __future__ import annotations + +import logging +import queue +import threading +import time +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full +_DRAIN_BATCH = 256 + + +class MonitoringEmitter: + """Owns the queue, the dispatcher thread, and the subscriber list.""" + + def __init__(self, *, enabled: bool = True) -> None: + self._enabled = enabled + self._q: "queue.Queue[Dict[str, Any]]" = queue.Queue(maxsize=_MAX_QUEUE) + self._dropped = 0 + self._dispatched = 0 + self._stop = threading.Event() + self._started = False + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + # Live subscribers (the OTLP streamers). Called from the dispatcher + # thread, fully fail-isolated. Each subscriber is callable(batch: list[dict]). + self._subscribers: list = [] + + # ── public API (hot path) ─────────────────────────────────────────────── + def emit(self, event: Any) -> None: + """Enqueue an event. Never blocks, never raises. + + ``event`` may be a dataclass with ``to_dict()`` or a plain dict. + """ + if not self._enabled: + return + try: + payload = event.to_dict() if hasattr(event, "to_dict") else dict(event) + payload.setdefault("ts_ns", time.time_ns()) + self._ensure_started() + try: + self._q.put_nowait(payload) + except queue.Full: + # Drop oldest to make room — bounded memory, newest-wins. + try: + self._q.get_nowait() + self._q.task_done() + self._dropped += 1 + self._q.put_nowait(payload) + except Exception: + self._dropped += 1 + except Exception: # the hot-path invariant: never propagate + logger.debug("monitoring emit failed", exc_info=True) + + # ── lifecycle ─────────────────────────────────────────────────────────── + def _ensure_started(self) -> None: + if self._started: + return + with self._lock: + if self._started: + return + self._thread = threading.Thread( + target=self._run, name="hermes-monitoring-dispatch", daemon=True + ) + self._thread.start() + self._started = True + + def _run(self) -> None: + while not self._stop.is_set(): + try: + first = self._q.get(timeout=0.5) + except queue.Empty: + continue + batch = [first] + while len(batch) < _DRAIN_BATCH: + try: + batch.append(self._q.get_nowait()) + except queue.Empty: + break + try: + self._dispatch(batch) + finally: + for _ in batch: + self._q.task_done() + + def _dispatch(self, batch) -> None: + # Fan-out to subscribers (OTLP streamers) — fully fail-isolated. + for sub in list(self._subscribers): + try: + sub(batch) + except Exception: + logger.debug("monitoring subscriber failed", exc_info=True) + self._dispatched += len(batch) + + def subscribe(self, callback) -> None: + """Register a live batch subscriber (callable(batch: list[dict])).""" + if callback not in self._subscribers: + self._subscribers.append(callback) + self._enabled = True + + def unsubscribe(self, callback) -> None: + try: + self._subscribers.remove(callback) + except ValueError: + pass + if not self._subscribers: + self._enabled = False + + # ── introspection / shutdown (tests, CLI) ─────────────────────────────── + def flush(self, timeout: float = 2.0) -> None: + """Wait boundedly for queued and in-flight batches to finish dispatch.""" + if timeout <= 0: + return + + finished = threading.Event() + + def _wait_for_completion() -> None: + self._q.join() + finished.set() + + waiter = threading.Thread( + target=_wait_for_completion, + name="hermes-monitoring-flush", + daemon=True, + ) + waiter.start() + finished.wait(timeout=timeout) + + def stats(self) -> Dict[str, int]: + return { + "queued": self._q.qsize(), + "dispatched": self._dispatched, + "dropped": self._dropped, + "subscribers": len(self._subscribers), + } + + def close(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._started = False + + +# ── process-wide singleton ────────────────────────────────────────────────── +_EMITTER: Optional[MonitoringEmitter] = None +_EMITTER_LOCK = threading.Lock() + + +def get_emitter() -> MonitoringEmitter: + """Return the process-wide monitoring emitter.""" + global _EMITTER + if _EMITTER is not None: + return _EMITTER + with _EMITTER_LOCK: + if _EMITTER is None: + # Collection is opt-in. A plane exporter enables the singleton by + # attaching its first subscriber; until then producers are no-ops. + _EMITTER = MonitoringEmitter(enabled=False) + return _EMITTER + + +def emit(event: Any) -> None: + """Module-level convenience: emit via the singleton.""" + get_emitter().emit(event) + + +def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None: + """Swap the singleton (tests only).""" + global _EMITTER + with _EMITTER_LOCK: + if _EMITTER is not None and emitter is not _EMITTER: + try: + _EMITTER.close() + except Exception: + pass + _EMITTER = emitter + + +# Back-compat alias for the salvaged class name used in emozilla's tests. +TelemetryEmitter = MonitoringEmitter + +__all__ = [ + "MonitoringEmitter", + "TelemetryEmitter", + "get_emitter", + "emit", + "reset_emitter_for_tests", +] diff --git a/agent/monitoring/events.py b/agent/monitoring/events.py new file mode 100644 index 000000000000..a17f4a49425d --- /dev/null +++ b/agent/monitoring/events.py @@ -0,0 +1,86 @@ +"""Typed gateway monitoring events. + +Content-free service-health and redacted diagnostic events for the gateway +daemon. These are the only event shapes the monitoring plane emits: no +prompts, messages, tool args/results, session history, or usage analytics. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field, asdict +from typing import Any, Dict, Optional + + +def _now_ns() -> int: + return time.time_ns() + + +@dataclass(slots=True) +class GatewayHealthEvent: + """Content-free gateway health snapshot or lifecycle event.""" + + name: str + gateway_state: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + exit_reason: Optional[str] = None + restart_requested: Optional[bool] = None + active_agents: int = 0 + gateway_busy: bool = False + gateway_drainable: bool = False + platform_count: int = 0 + fatal_platform_count: int = 0 + profile: Optional[str] = None + install_id: Optional[str] = None + version: Optional[str] = None + supervision_mode: Optional[str] = None + pid: Optional[int] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_health", **asdict(self)} + + +@dataclass(slots=True) +class GatewayDiagnosticEvent: + """Redacted gateway diagnostic event for operator-owned observability.""" + + name: str + subsystem: str + error_class: str = "unknown" + error_code: Optional[str] = None + platform: Optional[str] = None + old_state: Optional[str] = None + new_state: Optional[str] = None + profile: Optional[str] = None + version: Optional[str] = None + severity: str = "warning" + ts_ns: int = field(default_factory=_now_ns) + source_logger: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return {"event": "gateway_diagnostic", **asdict(self)} + + +@dataclass(slots=True) +class CronExecutionEvent: + """Content-free durable cron execution lifecycle projection.""" + + status: str + job_key: str + source: str = "unknown" + duration_ms: Optional[int] = None + delivery_outcome: Optional[str] = None + error_class: Optional[str] = None + ts_ns: int = field(default_factory=_now_ns) + + def to_dict(self) -> Dict[str, Any]: + return {"event": "cron_execution", **asdict(self)} + + +__all__ = [ + "GatewayHealthEvent", + "GatewayDiagnosticEvent", + "CronExecutionEvent", +] diff --git a/agent/monitoring/gateway_health.py b/agent/monitoring/gateway_health.py new file mode 100644 index 000000000000..752473fe31bb --- /dev/null +++ b/agent/monitoring/gateway_health.py @@ -0,0 +1,469 @@ +"""Gateway health and diagnostics signal producer. + +This module keeps the plane narrow: service health monitoring plus +redacted operational diagnostics. It reuses the existing gateway runtime-status +contract and emits content-free metrics/events. No prompts, messages, tool args, +session history, audit records, or product analytics belong here. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent + + +@dataclass(frozen=True, slots=True) +class GatewayMetric: + name: str + value: int | float + attributes: Dict[str, str] + + +@dataclass(frozen=True, slots=True) +class GatewayHealthSnapshot: + metrics: List[GatewayMetric] + events: List[GatewayHealthEvent | GatewayDiagnosticEvent] + + +_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"} +_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"} +_KNOWN_GATEWAY_STATES = { + "starting", "draining", "stopping", "stopped", "startup_failed", "unknown" +} | _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES +_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | { + "connecting", "disconnected", "disabled", "paused", "retrying", "unknown" +} +_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"} +_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$") + + +def _allowed_logger(name: str) -> bool: + return name == "gateway" or name.startswith("gateway.") + + +def source_logger_for_export(name: Any) -> Optional[str]: + """Return a bounded source-controlled gateway logger name for OTLP scope.""" + value = str(name or "") + return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None + + +def redact_gateway_message(message: Any) -> str: + """Redact gateway diagnostic free text for operator-owned export. + + Single scrub path: everything goes through + ``agent.monitoring.redaction.redact_for_export`` (unconditional + secrets + PII), then is length-bounded. + """ + try: + from agent.monitoring.redaction import redact_for_export + redacted = redact_for_export(str(message or "")) or "" + except Exception: + redacted = "[redaction-unavailable]" + return redacted[:500] + + +def classify_gateway_error(raw: Any) -> str: + s = str(raw or "").lower() + if any(k in s for k in ("auth", "token", "unauthorized", "forbidden", "401", "403")): + return "auth_failed" + if "rate" in s and "limit" in s: + return "rate_limited" + if "timeout" in s or "timed out" in s: + return "timeout" + if any( + k in s + for k in ( + "network", + "connection", + "dns", + "socket", + "connect call failed", + "failed to connect", + "cannot connect", + "unreachable", + "name resolution", + ) + ): + return "network_error" + if any(k in s for k in ("config", "missing", "invalid")): + return "invalid_config" + if "startup" in s: + return "startup_failed" + if "fatal" in s: + return "platform_fatal" + return "unknown" + + +def classify_exit_reason( + raw: Any, *, state: Any, restart_requested: bool +) -> Optional[str]: + """Reduce free-form shutdown text to a bounded operational class.""" + if restart_requested: + return "restart_requested" + state_name = str(state or "").lower() + if raw is None and state_name != "startup_failed": + return None + classified = classify_gateway_error(raw) + if state_name == "startup_failed": + return classified if classified != "unknown" else "startup_failed" + text = str(raw or "").lower() + if "signal" in text or "sigterm" in text or "sigint" in text: + return "signal" + if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")): + return "planned_stop" + return classified + + +def _bounded_state(raw: Any, *, allowed: set[str]) -> str: + state = str(raw or "unknown").lower() + return state if state in allowed else "unknown" + + +def _safe_metric_value(raw: Any, *, limit: int = 128) -> str: + try: + from agent.monitoring.redaction import redact_for_export + value = redact_for_export(str(raw or "")) or "unknown" + except Exception: + return "unknown" + return value[:limit] + + +def _safe_instance_id(raw: Any) -> str: + """Return a stable opaque instance key without exporting the source ID.""" + value = str(raw or "unknown").encode("utf-8", errors="replace") + return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}" + + +def subsystem_for_logger(logger_name: str) -> str: + if logger_name == "gateway.relay" or logger_name.startswith("gateway.relay."): + return "platform.relay" + if logger_name.startswith("gateway.platforms."): + parts = logger_name.split(".") + if len(parts) >= 3 and parts[2]: + return f"platform.{parts[2]}" + if logger_name.startswith("gateway.platforms"): + return "platform" + if logger_name.startswith("gateway"): + return "gateway" + return "gateway" + + +def platform_for_subsystem(subsystem: str) -> Optional[str]: + if subsystem.startswith("platform."): + return subsystem.split(".", 1)[1] or None + return None + + +def _parse_active_agents(raw: Any) -> int: + try: + from gateway.status import parse_active_agents + return parse_active_agents(raw) + except Exception: + try: + return max(0, int(raw)) + except (TypeError, ValueError): + return 0 + + +def _derive_busy(gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: + try: + from gateway.status import derive_gateway_busy + return derive_gateway_busy( + gateway_running=gateway_running, + gateway_state=gateway_state, + active_agents=active_agents, + ) + except Exception: + return bool(gateway_running and gateway_state == "running" and _parse_active_agents(active_agents) > 0) + + +def _derive_drainable(gateway_running: bool, gateway_state: Any) -> bool: + try: + from gateway.status import derive_gateway_drainable + return derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state) + except Exception: + return bool(gateway_running and gateway_state == "running") + + +def _base_attrs(*, profile: str, install_id: str, version: str, supervision_mode: str) -> Dict[str, str]: + mode = str(supervision_mode or "unknown").lower() + return { + "service.instance.id": _safe_instance_id(install_id), + "service.version": _safe_metric_value(version, limit=64), + "hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown", + } + + +def _metric(name: str, value: int | float, attrs: Dict[str, str], **extra: str) -> GatewayMetric: + out = dict(attrs) + for key, val in extra.items(): + if val is not None: + out[key] = _safe_metric_value(val) + return GatewayMetric(name=name, value=value, attributes=out) + + +def build_gateway_health_snapshot( + runtime: Optional[dict[str, Any]], + *, + gateway_running: bool, + profile: str, + install_id: str, + version: str, + supervision_mode: str = "unknown", +) -> GatewayHealthSnapshot: + """Convert gateway_state.json-compatible runtime state into P0 signals.""" + runtime = runtime or {} + gateway_state = _bounded_state( + runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) + active_agents = _parse_active_agents(runtime.get("active_agents", 0)) + busy = _derive_busy(gateway_running, gateway_state, active_agents) + drainable = _derive_drainable(gateway_running, gateway_state) + platforms = runtime.get("platforms") if isinstance(runtime.get("platforms"), dict) else {} + base = _base_attrs(profile=profile, install_id=install_id, version=version, supervision_mode=supervision_mode) + + metrics: list[GatewayMetric] = [ + _metric("hermes.gateway.up", 1 if gateway_running else 0, base), + _metric("hermes.gateway.active_agents", active_agents, base), + _metric("hermes.gateway.busy", 1 if busy else 0, base), + _metric("hermes.gateway.drainable", 1 if drainable else 0, base), + _metric("hermes.gateway.restart_requested", 1 if runtime.get("restart_requested") else 0, base), + ] + if gateway_state: + metrics.append(_metric("hermes.gateway.state", 1, base, **{"hermes.gateway.state": str(gateway_state)})) + + fatal_count = 0 + events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + for platform, pdata in platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) + raw_error = pdata.get("error_code") or pdata.get("error_message") + error_code = classify_gateway_error(raw_error) + is_up = state in _RUNNING_PLATFORM_STATES + is_degraded = state in _FATAL_PLATFORM_STATES + if is_degraded: + fatal_count += 1 + metrics.append(_metric( + "hermes.platform.up", + 1 if is_up else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state}, + )) + metrics.append(_metric( + "hermes.platform.degraded", + 1 if is_degraded else 0, + base, + **{"hermes.platform": str(platform), "hermes.platform.state": state, "hermes.error_code": error_code}, + )) + if is_degraded: + events.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=classify_gateway_error(error_code or pdata.get("error_message")), + profile=profile, + version=version, + severity="error" if state == "fatal" else "warning", + )) + + events.insert(0, GatewayHealthEvent( + name="gateway.health_snapshot", + gateway_state=str(gateway_state) if gateway_state is not None else None, + active_agents=active_agents, + gateway_busy=busy, + gateway_drainable=drainable, + platform_count=len(platforms), + fatal_platform_count=fatal_count, + profile=profile, + install_id=install_id, + version=version, + supervision_mode=supervision_mode, + pid=_coerce_pid(runtime.get("pid")), + )) + return GatewayHealthSnapshot(metrics=metrics, events=events) + + +def _safe_profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _safe_version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None: + """Emit immediate content-free gateway events for runtime status changes. + + Called by gateway.status.write_runtime_status after persisting the new status. + Fully fail-open: failures never affect gateway status writes. + """ + try: + from agent.monitoring import emitter + out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [] + profile = _safe_profile() + version = _safe_version() + old_gateway_state = _bounded_state( + (previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if (previous or {}).get("gateway_state") is not None else None + new_gateway_state = _bounded_state( + current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES + ) if current.get("gateway_state") is not None else None + if old_gateway_state != new_gateway_state and new_gateway_state: + out.append(GatewayHealthEvent( + name="gateway.lifecycle", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=classify_exit_reason( + current.get("exit_reason"), + state=new_gateway_state, + restart_requested=bool(current.get("restart_requested")), + ), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + if new_gateway_state == "startup_failed": + out.append(GatewayDiagnosticEvent( + name="gateway.startup_failed", + subsystem="gateway", + error_class=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + error_code=classify_gateway_error(current.get("exit_reason") or "startup_failed"), + profile=profile, + version=version, + severity="error", + )) + if new_gateway_state == "stopped": + out.append(GatewayHealthEvent( + name="gateway.exit", + gateway_state=new_gateway_state, + old_state=old_gateway_state, + new_state=new_gateway_state, + exit_reason=classify_exit_reason( + current.get("exit_reason"), + state=new_gateway_state, + restart_requested=bool(current.get("restart_requested")), + ), + restart_requested=bool(current.get("restart_requested")), + active_agents=_parse_active_agents(current.get("active_agents", 0)), + profile=profile, + version=version, + pid=_coerce_pid(current.get("pid")), + )) + + old_platforms_raw = (previous or {}).get("platforms") + new_platforms_raw = current.get("platforms") + old_platforms = old_platforms_raw if isinstance(old_platforms_raw, dict) else {} + new_platforms = new_platforms_raw if isinstance(new_platforms_raw, dict) else {} + for platform, pdata in new_platforms.items(): + pdata = pdata if isinstance(pdata, dict) else {} + prev_raw = old_platforms.get(platform, {}) + prev = prev_raw if isinstance(prev_raw, dict) else {} + old_state = _bounded_state( + prev.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if prev.get("state") is not None else None + new_state = _bounded_state( + pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES + ) if pdata.get("state") is not None else None + if old_state == new_state or not new_state: + continue + error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message")) + severity = "error" if new_state.lower() in {"fatal", "failed", "error"} else "warning" + out.append(GatewayDiagnosticEvent( + name="platform.state_change", + subsystem=f"platform.{platform}", + platform=str(platform), + old_state=old_state, + new_state=new_state, + error_code=error_code, + error_class=error_code, + profile=profile, + version=version, + severity=severity, + )) + if new_state.lower() in _FATAL_PLATFORM_STATES: + out.append(GatewayDiagnosticEvent( + name="platform.fatal", + subsystem=f"platform.{platform}", + platform=str(platform), + error_code=error_code, + error_class=error_code, + profile=profile, + version=version, + severity=severity, + )) + for ev in out: + emitter.emit(ev) + except Exception: + logging.getLogger(__name__).debug("gateway runtime status transition emit failed", exc_info=True) + + +def _coerce_pid(raw: Any) -> Optional[int]: + try: + pid = int(raw) + except (TypeError, ValueError): + return None + return pid if pid > 0 else None + + +class GatewayDiagnosticLogHandler(logging.Handler): + """Allowlisted warning/error bridge for gateway-owned diagnostics.""" + + def __init__(self, *, profile: str = "default", version: str = "unknown") -> None: + super().__init__(level=logging.WARNING) + self.profile = profile + self.version = version + + def emit(self, record: logging.LogRecord) -> None: + try: + if record.levelno < logging.WARNING: + return + if not _allowed_logger(record.name): + return + subsystem = subsystem_for_logger(record.name) + message = record.getMessage() + error_class = classify_gateway_error(message) + event = GatewayDiagnosticEvent( + name=f"gateway.log.{record.levelname.lower()}", + subsystem=subsystem, + source_logger=source_logger_for_export(record.name), + platform=platform_for_subsystem(subsystem), + error_class=error_class, + error_code=error_class, + profile=self.profile, + version=self.version, + severity=record.levelname.lower(), + ) + from agent.monitoring import emitter + emitter.get_emitter().emit(event) + except Exception: + logging.getLogger(__name__).debug("gateway diagnostic emit failed", exc_info=True) + + +__all__ = [ + "GatewayMetric", + "GatewayHealthSnapshot", + "GatewayDiagnosticLogHandler", + "build_gateway_health_snapshot", + "classify_gateway_error", + "source_logger_for_export", + "redact_gateway_message", +] diff --git a/agent/monitoring/gateway_health_export.py b/agent/monitoring/gateway_health_export.py new file mode 100644 index 000000000000..0a377c99ab30 --- /dev/null +++ b/agent/monitoring/gateway_health_export.py @@ -0,0 +1,643 @@ +"""Gateway Health & Diagnostics OTLP export runtime. + +This exporter emits operator-owned gateway service-health metrics plus +narrow redacted diagnostic events. It is deliberately in-process and fail-open so +it works under systemd, launchd, s6, containers, tmux, nohup, or a simple shell +without a sidecar/watchdog dependency. +""" + +from __future__ import annotations + +import logging +import os +import re +import threading +from dataclasses import dataclass +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_DEFAULT_DIAGNOSTIC_SCOPE = "hermes.gateway.diagnostics" + +_RESOURCE_ATTRIBUTE_KEYS = frozenset({ + "service.name", + "service.namespace", + "service.version", + "service.instance.id", + "deployment.environment.name", + "cloud.provider", + "cloud.platform", + "cloud.region", + "telemetry.scope", +}) +_DIAGNOSTIC_ATTRIBUTE_KEYS = frozenset({ + "name", + "subsystem", + "error_class", + "error_code", + "platform", + "old_state", + "new_state", + "version", + "severity", +}) +_SAFE_RESOURCE_VALUE = re.compile(r"^[A-Za-z0-9._:/-]{1,128}$") + + +def _redact_string(raw: Any, *, limit: int = 500) -> str: + try: + from agent.monitoring.redaction import redact_for_export + return (redact_for_export(str(raw or "")) or "[redacted]")[:limit] + except Exception: + return "[redaction-unavailable]" + + +def _safe_resource_attributes(raw: Any) -> Dict[str, str]: + """Allowlist bounded resource labels and reject values changed by redaction.""" + attrs: Dict[str, str] = {} + if not isinstance(raw, dict): + return attrs + for key, value in raw.items(): + key = str(key) + if key not in _RESOURCE_ATTRIBUTE_KEYS or value is None: + continue + if key == "service.instance.id": + from agent.monitoring.gateway_health import _safe_instance_id + attrs[key] = _safe_instance_id(value) + continue + text = str(value) + if not _SAFE_RESOURCE_VALUE.fullmatch(text): + continue + if _redact_string(text, limit=128) != text: + continue + attrs[key] = text + return attrs + + +def _runtime_resource_attributes( + config: Dict[str, Any], *, telemetry_scope: str +) -> Dict[str, str]: + """Build the safe OTLP resource shared by metrics and diagnostic logs.""" + gh = _gateway_health_config(config) + attrs = _safe_resource_attributes(gh.get("resource_attributes")) + from agent.monitoring.gateway_health import _safe_instance_id + + attrs["service.name"] = "hermes-gateway" + attrs["service.instance.id"] = _safe_instance_id(_install_id(config)) + attrs["telemetry.scope"] = telemetry_scope + return attrs + + +def _diagnostic_log_attributes(event: Dict[str, Any]) -> Dict[str, Any]: + attrs: Dict[str, Any] = {} + for key in _DIAGNOSTIC_ATTRIBUTE_KEYS: + value = event.get(key) + if value is None: + continue + attrs[f"hermes.{key}"] = _redact_string(value) if isinstance(value, str) else value + return attrs + + +@dataclass(slots=True) +class GatewayHealthExportRuntime: + enabled: bool + reason: str = "disabled" + streamer: Any = None + metric_provider: Any = None + log_handler: Any = None + log_streamer: Any = None + thread: Optional[threading.Thread] = None + stop_event: Optional[threading.Event] = None + + def shutdown(self) -> None: + if self.stop_event is not None: + self.stop_event.set() + if self.thread is not None: + self.thread.join(timeout=0.25) + if self.log_handler is not None: + try: + logging.getLogger().removeHandler(self.log_handler) + except Exception: + pass + + # All producers above are now stopped. Drain queued and in-flight + # events before detaching subscribers so the terminal lifecycle event + # cannot race exporter shutdown. The barrier is bounded and fail-open. + try: + from agent.monitoring.emitter import get_emitter + emitter = get_emitter() + emitter.flush(timeout=1.0) + if self.streamer is not None: + emitter.unsubscribe(self.streamer) + if self.log_streamer is not None: + emitter.unsubscribe(self.log_streamer) + except Exception: + pass + + # Network flush/close runs under one bounded daemon-thread deadline and + # can never delay gateway teardown indefinitely. + closeables = [ + item for item in (self.streamer, self.log_streamer, self.metric_provider) + if item is not None + ] + + def _close() -> None: + for item in closeables: + try: + item.shutdown() + except Exception: + pass + + if closeables: + worker = threading.Thread( + target=_close, + name="hermes-gateway-health-export-shutdown", + daemon=True, + ) + worker.start() + worker.join(timeout=2.0) + + self.streamer = None + self.log_streamer = None + self.metric_provider = None + self.thread = None + self.stop_event = None + + +def _gateway_health_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + return mon.get("gateway_health_export") or {} + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} + return export.get("otlp") or {} + + +def _enabled(config: Dict[str, Any]) -> bool: + gh = _gateway_health_config(config) + otlp = _otlp_config(config) + return bool(gh.get("enabled") and otlp.get("enabled") and otlp.get("endpoint")) + + +def _require_metrics_sdk(*, auto_install: bool = True, prompt: bool = False) -> Dict[str, Any]: + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except Exception: + pass + try: + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.metrics import Observation + from opentelemetry.trace import INVALID_SPAN_ID, INVALID_TRACE_ID, TraceFlags + from opentelemetry._logs import LogRecord + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + return { + "OTLPLogExporter": OTLPLogExporter, + "OTLPMetricExporter": OTLPMetricExporter, + "Observation": Observation, + "LogRecord": LogRecord, + "LoggerProvider": LoggerProvider, + "INVALID_SPAN_ID": INVALID_SPAN_ID, + "INVALID_TRACE_ID": INVALID_TRACE_ID, + "TraceFlags": TraceFlags, + "SeverityNumber": SeverityNumber, + "BatchLogRecordProcessor": BatchLogRecordProcessor, + "MeterProvider": MeterProvider, + "PeriodicExportingMetricReader": PeriodicExportingMetricReader, + "Resource": Resource, + } + except Exception as exc: + raise RuntimeError(f"OTLP metrics SDK unavailable: {exc}") from exc + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + return resolved + + +def _metric_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/metrics" + return endpoint + + +def _logs_endpoint(endpoint: str) -> str: + if endpoint.endswith("/v1/traces"): + return endpoint[: -len("/v1/traces")] + "/v1/logs" + if endpoint.endswith("/v1/metrics"): + return endpoint[: -len("/v1/metrics")] + "/v1/logs" + return endpoint + + +def _version() -> str: + try: + from hermes_cli import __version__ + return str(__version__) + except Exception: + return "unknown" + + +def _profile() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return str(get_active_profile_name() or "default") + except Exception: + return "default" + + +def _install_id(config: Dict[str, Any]) -> str: + try: + from agent.monitoring.policy import ensure_install_id + return str(ensure_install_id(config)) + except Exception: + return "unknown" + + +def _supervision_mode() -> str: + if os.environ.get("INVOCATION_ID"): + return "systemd" + if os.environ.get("S6_CMD_ARG0") or os.environ.get("S6_VERSION"): + return "s6" + if os.environ.get("container") or os.path.exists("/.dockerenv"): + return "container" + if os.environ.get("LAUNCHD_SOCKET"): + return "launchd" + return "manual" + + +def _read_gateway_snapshot(config: Dict[str, Any]): + from agent.monitoring.gateway_health import build_gateway_health_snapshot + try: + from gateway.status import read_runtime_status + runtime = read_runtime_status() or {} + except Exception: + runtime = {} + return build_gateway_health_snapshot( + runtime, + gateway_running=True, + profile=_profile(), + install_id=_install_id(config), + version=_version(), + supervision_mode=_supervision_mode(), + ) + + +def _read_cron_snapshot(): + from agent.monitoring.cron_health import build_cron_health_snapshot + + return build_cron_health_snapshot() + + +def _read_background_work_count() -> int: + """Count live background/subagent work that ``active_agents`` does NOT include. + + ``hermes.gateway.active_agents`` counts foreground turns + in-flight cron + jobs + API runs, but deliberately excludes backgrounded ``delegate_task`` + subagents, ``terminal(background=true)`` processes, kanban workers, and the + runner's own background tasks (they are tracked only for the scale-to-zero + suspend guard, ``_scale_to_zero_has_live_background_work``). Without this + metric a peer churning through delegated subagents shows ``active_agents=0`` + on the fleet dashboard. Best-effort and content-free: a single integer, + no job/task identity. Returns 0 if a source can't be imported. + + Delegation is counted TASK-granular (``active_task_count``): a fan-out batch + of N subagents contributes N, not 1, so the metric reflects real concurrent + subagent load rather than dispatch-unit/pool-slot count. This intentionally + differs from the async pool's capacity accounting (one batch = one slot). + """ + total = 0 + try: + from tools.async_delegation import active_task_count + + total += max(0, int(active_task_count())) + except Exception: + logger.debug("background-work async-delegation count failed", exc_info=True) + try: + from tools.process_registry import process_registry + + total += max(0, int(process_registry.count_running())) + except Exception: + logger.debug("background-work process-registry count failed", exc_info=True) + return total + + +def _read_background_delegations_count() -> int: + """Count live async delegation UNITS (dispatch/pool slots). + + Complements ``_read_background_work_count`` (which is task-granular): this + counts each ``delegate_task`` dispatch as ONE regardless of fan-out width, + matching the async pool's capacity accounting (a batch = one slot). Together + the two metrics let an operator see both slot pressure + (``background_delegations``, alert vs ``max_concurrent_children``) and real + concurrent subagent load (``background_work``). Delegations only — it does + not include ``terminal(background)`` / kanban work, which are already folded + into ``background_work``. Best-effort; 0 if the source can't be imported. + """ + try: + from tools.async_delegation import active_count + + return max(0, int(active_count())) + except Exception: + logger.debug("background-delegations count failed", exc_info=True) + return 0 + + +def _read_runtime_snapshot(config: Dict[str, Any]): + gateway_snapshot = _read_gateway_snapshot(config) + # Background/subagent work — a distinct metric from active_agents (which + # never counts it). Appended to the gateway snapshot so it rides the same + # base resource attributes (service.instance.id etc.). + try: + from agent.monitoring.gateway_health import GatewayMetric + + base = dict(gateway_snapshot.metrics[0].attributes) if gateway_snapshot.metrics else {} + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_work", + value=_read_background_work_count(), + attributes=base, + ) + ) + gateway_snapshot.metrics.append( + GatewayMetric( + name="hermes.gateway.background_delegations", + value=_read_background_delegations_count(), + attributes=base, + ) + ) + except Exception as exc: + logger.warning( + "background-work snapshot unavailable; metric not exported (error_type=%s)", + type(exc).__name__, + ) + logger.debug("background-work snapshot traceback", exc_info=True) + try: + cron_snapshot = _read_cron_snapshot() + except Exception as exc: + # Content-free visibility: cron telemetry silently dropping out is a + # release-relevant regression, so surface it at WARNING with only the + # exception *type* name (never the message, which could carry paths or + # other environment detail). exc_info stays on the DEBUG record. + logger.warning( + "cron health snapshot unavailable; cron telemetry not exported (error_type=%s)", + type(exc).__name__, + ) + logger.debug("cron health snapshot traceback", exc_info=True) + return gateway_snapshot + gateway_snapshot.metrics.extend(cron_snapshot.metrics) + return gateway_snapshot + + +def _emit_snapshot_events(config: Dict[str, Any]) -> None: + gh = _gateway_health_config(config) + if not gh.get("diagnostic_events_enabled", True): + return + try: + from agent.monitoring import emitter + snapshot = _read_runtime_snapshot(config) + for event in snapshot.events: + emitter.emit(event) + except Exception: + logger.debug("gateway health snapshot emit failed", exc_info=True) + + +def _start_metric_provider(config: Dict[str, Any], sdk: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("metrics_enabled", True): + return None + otlp = _otlp_config(config) + endpoint = _metric_endpoint(str(otlp.get("endpoint"))) + headers = _resolve_headers(otlp.get("headers_env")) + exporter = sdk["OTLPMetricExporter"](endpoint=endpoint, headers=headers or None) + interval_ms = max(5, int(gh.get("export_interval_seconds", 60))) * 1000 + reader = sdk["PeriodicExportingMetricReader"](exporter, export_interval_millis=interval_ms) + resource_attrs = _runtime_resource_attributes( + config, telemetry_scope="gateway_health" + ) + provider = sdk["MeterProvider"]( + metric_readers=[reader], + resource=sdk["Resource"].create(resource_attrs), + ) + meter = provider.get_meter("hermes.gateway.health") + Observation = sdk["Observation"] + + metric_names = [ + "hermes.gateway.up", + "hermes.gateway.state", + "hermes.gateway.active_agents", + "hermes.gateway.busy", + "hermes.gateway.drainable", + "hermes.gateway.restart_requested", + "hermes.gateway.background_work", + "hermes.gateway.background_delegations", + "hermes.platform.up", + "hermes.platform.degraded", + "hermes.cron.scheduler.heartbeat_age_seconds", + "hermes.cron.scheduler.last_success_age_seconds", + "hermes.cron.scheduler.catch_up_occurrences", + "hermes.cron.jobs.enabled", + "hermes.cron.jobs.running", + "hermes.cron.jobs.overdue", + ] + + def callback(name: str): + def _cb(_options=None): + try: + snapshot = _read_runtime_snapshot(config) + return [Observation(m.value, m.attributes) for m in snapshot.metrics if m.name == name] + except Exception: + logger.debug("gateway metric callback failed", exc_info=True) + return [] + return _cb + + for metric_name in metric_names: + meter.create_observable_gauge(metric_name, callbacks=[callback(metric_name)]) + return provider + + +def _severity_number(sdk: Dict[str, Any], severity: Any) -> Any: + SeverityNumber = sdk["SeverityNumber"] + sev = str(severity or "warning").lower() + if sev in {"critical", "fatal"}: + return SeverityNumber.FATAL + if sev == "error": + return SeverityNumber.ERROR + if sev in {"info", "information"}: + return SeverityNumber.INFO + if sev == "debug": + return SeverityNumber.DEBUG + return SeverityNumber.WARN + + +class GatewayDiagnosticLogStreamer: + """Emitter subscriber that sends gateway diagnostic events as OTLP logs.""" + + def __init__(self, config: Dict[str, Any], sdk: Dict[str, Any]): + otlp = _otlp_config(config) + headers = _resolve_headers(otlp.get("headers_env")) + endpoint = _logs_endpoint(str(otlp.get("endpoint"))) + resource_attrs = _runtime_resource_attributes( + config, telemetry_scope="gateway_diagnostics" + ) + self._provider = sdk["LoggerProvider"](resource=sdk["Resource"].create(resource_attrs)) + self._processor = sdk["BatchLogRecordProcessor"]( + sdk["OTLPLogExporter"](endpoint=endpoint, headers=headers or None) + ) + self._provider.add_log_record_processor(self._processor) + self._logger = self._provider.get_logger(_DEFAULT_DIAGNOSTIC_SCOPE) + self._LogRecord = sdk["LogRecord"] + self._sdk = sdk + self.exported = 0 + + def __call__(self, batch: list[Dict[str, Any]]) -> None: + from agent.monitoring.gateway_health import source_logger_for_export + + for ev in batch: + if ev.get("event") != "gateway_diagnostic": + continue + attrs = _diagnostic_log_attributes(ev) + # Preserve the source-controlled Python logger as the OTel + # instrumentation scope. This adds precise code attribution without + # turning a fluid module layout into a maintained subsystem enum. + # Rendered messages stay out because they may contain arbitrary IDs, + # names, paths, or configured strings. A future, separately gated + # ``diagnostic_detail: redacted_message`` mode may add best-effort + # free text when an observability plane defines that privacy policy. + source_logger = source_logger_for_export(ev.get("source_logger")) + otel_logger = ( + self._provider.get_logger(source_logger) + if source_logger is not None + else self._logger + ) + body = "gateway diagnostic" + record = self._LogRecord( + timestamp=ev.get("ts_ns"), + trace_id=self._sdk["INVALID_TRACE_ID"], + span_id=self._sdk["INVALID_SPAN_ID"], + trace_flags=self._sdk["TraceFlags"].DEFAULT, + severity_text=str(ev.get("severity") or "warning").upper(), + severity_number=_severity_number(self._sdk, ev.get("severity")), + body=_redact_string(body), + attributes=attrs, + ) + otel_logger.emit(record) + self.exported += 1 + + def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def _start_diagnostic_log_streamer(config: Dict[str, Any], sdk: Dict[str, Any]) -> GatewayDiagnosticLogStreamer: + from agent.monitoring.emitter import get_emitter + streamer = GatewayDiagnosticLogStreamer(config, sdk) + get_emitter().subscribe(streamer) + return streamer + + +def _start_snapshot_thread(config: Dict[str, Any], stop_event: threading.Event) -> threading.Thread: + interval = max(5, int(_gateway_health_config(config).get("logs_export_interval_seconds", 5))) + + def _run() -> None: + while not stop_event.wait(interval): + _emit_snapshot_events(config) + + thread = threading.Thread(target=_run, name="hermes-gateway-health-export", daemon=True) + thread.start() + return thread + + +def _attach_log_handler(config: Dict[str, Any]) -> Any: + gh = _gateway_health_config(config) + if not gh.get("diagnostic_events_enabled", True) or not gh.get("warning_error_events_enabled", True): + return None + from agent.monitoring.gateway_health import GatewayDiagnosticLogHandler + handler = GatewayDiagnosticLogHandler(profile=_profile(), version=_version()) + root = logging.getLogger() + if handler not in root.handlers: + root.addHandler(handler) + return handler + + +def _gateway_health_event(ev: Dict[str, Any]) -> bool: + return ev.get("event") in {"gateway_health", "cron_execution"} + + +def start_gateway_health_export(config: Dict[str, Any]) -> GatewayHealthExportRuntime: + """Start P0 gateway health export if configured. Never raises.""" + if not _enabled(config): + return GatewayHealthExportRuntime(enabled=False, reason="disabled") + gh = _gateway_health_config(config) + runtime = GatewayHealthExportRuntime(enabled=True, reason="enabled") + sdk: Optional[Dict[str, Any]] = None + + if gh.get("metrics_enabled", True) or gh.get("diagnostic_events_enabled", True): + try: + sdk = _require_metrics_sdk(prompt=False) + except Exception: + logger.warning( + "monitoring.gateway_health_export.enabled but OTLP SDK is unavailable; " + "install 'hermes-agent[otlp]'", + exc_info=True, + ) + return GatewayHealthExportRuntime(enabled=False, reason="otlp_unavailable") + + if gh.get("metrics_enabled", True) and sdk is not None: + try: + runtime.metric_provider = _start_metric_provider(config, sdk) + except Exception: + logger.warning("gateway health OTLP metrics failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="metrics_start_failed") + + if gh.get("diagnostic_events_enabled", True) and sdk is not None: + try: + from agent.monitoring import otlp_exporter + runtime.streamer = otlp_exporter.start_streaming(config, event_filter=_gateway_health_event) + if runtime.streamer is None: + raise RuntimeError("gateway health span streamer did not start") + runtime.log_streamer = _start_diagnostic_log_streamer(config, sdk) + except Exception: + logger.debug("gateway diagnostic OTLP export failed to start", exc_info=True) + runtime.shutdown() + return GatewayHealthExportRuntime(enabled=False, reason="diagnostics_start_failed") + + try: + runtime.log_handler = _attach_log_handler(config) + except Exception: + logger.debug("gateway diagnostic log handler failed to attach", exc_info=True) + if gh.get("diagnostic_events_enabled", True): + try: + _emit_snapshot_events(config) + runtime.stop_event = threading.Event() + runtime.thread = _start_snapshot_thread(config, runtime.stop_event) + except Exception: + logger.debug("gateway health snapshot thread failed to start", exc_info=True) + return runtime + + +__all__ = [ + "GatewayHealthExportRuntime", + "start_gateway_health_export", +] diff --git a/agent/monitoring/otlp_exporter.py b/agent/monitoring/otlp_exporter.py new file mode 100644 index 000000000000..cde95673ec5f --- /dev/null +++ b/agent/monitoring/otlp_exporter.py @@ -0,0 +1,272 @@ +"""Export monitoring events to an OpenTelemetry Collector over OTLP/HTTP. + +Maps gateway monitoring events to OTel spans and sends them to the endpoint +configured under ``monitoring.export.otlp``. Lets an operator stream Hermes +gateway health into their own observability stack (OTEL Collector, DataDog, +and similar). + +Notes: + * The destination is operator-configured; this module only sends to that + endpoint. No default destination ships. + * ``opentelemetry-sdk`` + ``opentelemetry-exporter-otlp-proto-http`` are an + optional extra (``pip install hermes-agent[otlp]``), imported lazily so the + dependency is only required when OTLP export is actually used. + * ``headers_env`` maps a header name to an environment variable name; values + are read from the environment at export time and never logged or stored. + * The continuous subscriber runs in the emitter's dispatcher thread and is + fail-isolated, so an export error cannot affect the gateway. + +Only monitoring events (gateway_health / gateway_diagnostic) exist on this +plane; the ``event_filter`` seam is kept so future planes sharing the emitter +cannot silently ride along on this exporter. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Callable, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class OTLPUnavailable(RuntimeError): + """Raised when the optional OpenTelemetry SDK isn't installed.""" + + +def _require_sdk(*, auto_install: bool = True, prompt: bool = True): + """Import the OTel SDK, lazily installing it on first use if needed. + + Routes through tools.lazy_deps (feature 'export.otlp') so a missing SDK + triggers the standard venv install flow — same as every other optional + backend — gated by security.allow_lazy_installs and TTY-prompted. Falls back + to OTLPUnavailable (with a manual install hint) when the SDK can't be made + importable (lazy installs disabled, install failed, or auto_install=False). + + ``auto_install``: attempt the lazy install when missing (default True). + ``prompt``: ask before installing when interactive (default True); pass + False from non-interactive contexts like the continuous streamer. + """ + if auto_install: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("export.otlp", prompt=prompt) + except ImportError: + pass # lazy_deps unavailable — fall through to the import attempt + except Exception: + # FeatureUnavailable (lazy installs disabled / declined / failed) — + # fall through; the import below raises OTLPUnavailable with the hint. + pass + try: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.resources import Resource + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.trace import SpanKind + return { + "TracerProvider": TracerProvider, + "BatchSpanProcessor": BatchSpanProcessor, + "Resource": Resource, + "OTLPSpanExporter": OTLPSpanExporter, + "SpanKind": SpanKind, + } + except Exception as e: # ImportError or partial install + raise OTLPUnavailable( + "OTLP export requires the optional dependency. Install with:\n" + " pip install 'hermes-agent[otlp]'\n" + f"(import error: {e})" + ) + + +def _resolve_headers(headers_env: Optional[Dict[str, str]]) -> Dict[str, str]: + """Resolve {header_name: ENV_VAR_NAME} -> {header_name: value} from env. + + The config stores environment variable names, not secret values; values are + read from the environment here. Missing variables are skipped (and noted at + debug level without the value). + """ + resolved: Dict[str, str] = {} + for header_name, env_name in (headers_env or {}).items(): + val = os.environ.get(str(env_name)) + if val: + resolved[str(header_name)] = val + else: + logger.debug("OTLP header %s: env var %s not set; skipping", + header_name, env_name) + return resolved + + +def _otlp_config(config: Dict[str, Any]) -> Dict[str, Any]: + mon = (config or {}).get("monitoring") or {} + export = mon.get("export") or {} + return export.get("otlp") or {} + + +def build_exporter(config: Dict[str, Any]): + """Construct an OTLP span exporter from config. Raises OTLPUnavailable if no SDK.""" + sdk = _require_sdk() + otlp = _otlp_config(config) + endpoint = otlp.get("endpoint") + if not endpoint: + raise ValueError("monitoring.export.otlp.endpoint is not set") + headers = _resolve_headers(otlp.get("headers_env")) + return sdk["OTLPSpanExporter"](endpoint=endpoint, headers=headers or None) + + +def _resource_attributes(config: Dict[str, Any]) -> Dict[str, str]: + from agent.monitoring.gateway_health import _safe_instance_id + from agent.monitoring.policy import ensure_install_id + + return { + "service.name": "hermes-gateway", + "service.instance.id": _safe_instance_id(ensure_install_id(config)), + "telemetry.scope": "gateway_monitoring", + } + + +def _make_provider(config: Dict[str, Any]): + sdk = _require_sdk() + resource = sdk["Resource"].create(_resource_attributes(config)) + provider = sdk["TracerProvider"](resource=resource) + processor = sdk["BatchSpanProcessor"](build_exporter(config)) + provider.add_span_processor(processor) + return provider, processor + + +# ── event -> span attribute mapping ────────────────────────────────────────── +def _span_attrs(ev: Dict[str, Any]) -> Dict[str, Any]: + """Span attributes for a monitoring event (content-free by construction).""" + kind = ev.get("event") + attrs: Dict[str, Any] = {"hermes.event": kind or "unknown"} + keep_by_kind = { + "gateway_health": ("name", "gateway_state", "old_state", "new_state", + "exit_reason", "restart_requested", "active_agents", + "gateway_busy", "gateway_drainable", "platform_count", + "fatal_platform_count", "version", + "supervision_mode", "pid"), + "gateway_diagnostic": ("name", "subsystem", "error_class", "error_code", + "platform", "old_state", "new_state", + "version", "severity"), + "cron_execution": ("status", "job_key", "source", "duration_ms", + "delivery_outcome", "error_class"), + } + for col in keep_by_kind.get(kind, ()): # type: ignore[arg-type] + v = ev.get(col) + if v is not None: + if isinstance(v, str): + try: + from agent.monitoring.redaction import redact_for_export + v = (redact_for_export(v) or "[redacted]")[:500] + except Exception: + v = "[redaction-unavailable]" + attrs[f"hermes.{col}"] = v + return attrs + + +def export_batch(provider, batch: List[Dict[str, Any]]) -> int: + """Map a batch of events to OTel spans. Returns spans created.""" + tracer = provider.get_tracer("hermes.monitoring") + n = 0 + for ev in batch: + try: + name = f"hermes.{ev.get('event', 'event')}" + span = tracer.start_span(name, attributes=_span_attrs(ev)) + span.end() + n += 1 + except Exception: + logger.debug("OTLP span map failed", exc_info=True) + return n + + +# ── continuous streaming subscriber ───────────────────────────────────────── +class OTLPStreamer: + """A live subscriber that pushes each emitter batch to OTLP as it lands. + + Register with ``emitter.subscribe(streamer)``. Fail-isolated by the emitter. + """ + + def __init__( + self, + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, + ): + self._provider, self._processor = _make_provider(config) + self._event_filter = event_filter + self.exported = 0 + + def __call__(self, batch: List[Dict[str, Any]]) -> None: + if self._event_filter is not None: + batch = [ev for ev in batch if self._event_filter(ev)] + if not batch: + return + self.exported += export_batch(self._provider, batch) + + def shutdown(self) -> None: + try: + from agent.monitoring.emitter import get_emitter + get_emitter().unsubscribe(self) + except Exception: + pass + try: + self._processor.force_flush() + self._provider.shutdown() + except Exception: + pass + + +def is_available() -> bool: + """True when the OTel SDK is already importable. Does NOT auto-install — + this is a pure check (e.g. for status display).""" + try: + _require_sdk(auto_install=False) + return True + except OTLPUnavailable: + return False + + +def is_enabled(config: Dict[str, Any]) -> bool: + otlp = _otlp_config(config) + return bool(otlp.get("enabled") and otlp.get("endpoint")) + + +def start_streaming( + config: Dict[str, Any], + *, + event_filter: Optional[Callable[[Dict[str, Any]], bool]] = None, +) -> Optional[OTLPStreamer]: + """If OTLP is enabled, attach a streamer to the singleton emitter. + + ``event_filter`` scopes the exporter to its plane, e.g. gateway-health + export, so enabling one plane cannot silently export unrelated events. + + Non-interactive context (startup): attempts a lazy install with prompt=False + so a configured-but-missing SDK is installed once (gated by + security.allow_lazy_installs), then streams. If it still can't load, logs and + no-ops — never blocks or raises into startup. + """ + if not is_enabled(config): + return None + try: + _require_sdk(prompt=False) + except OTLPUnavailable: + logger.warning("monitoring.export.otlp.enabled but the OTel SDK could not " + "be installed/imported; install 'hermes-agent[otlp]'") + return None + from agent.monitoring.emitter import get_emitter + streamer = OTLPStreamer(config, event_filter=event_filter) + get_emitter().subscribe(streamer) + return streamer + + +__all__ = [ + "OTLPUnavailable", + "OTLPStreamer", + "build_exporter", + "export_batch", + "is_available", + "is_enabled", + "start_streaming", +] diff --git a/agent/monitoring/policy.py b/agent/monitoring/policy.py new file mode 100644 index 000000000000..1a42ce195caa --- /dev/null +++ b/agent/monitoring/policy.py @@ -0,0 +1,57 @@ +"""Install identity for gateway monitoring. + +The install id is a stable, resettable pseudonymous identifier attached to +exported health signals so an operator can tell instances apart in their +collector. It carries no account identity and can be rotated by clearing +``monitoring.install_id`` in config. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any, Dict + +logger = logging.getLogger(__name__) + + +def ensure_install_id(config: Dict[str, Any]) -> str: + """Return a stable install id, minting and persisting one when empty. + + The id must survive gateway restarts (it becomes ``service.instance.id`` + on exported signals), so a freshly minted UUID is written back to + config.yaml immediately. The write is fail-open: if persisting fails + (read-only home, managed scope), the ephemeral id is still returned and + a new one is minted next start. + + Clearing ``monitoring.install_id`` (e.g. ``hermes config set + monitoring.install_id ""``) rotates the id on the next gateway start. + """ + mon = config.get("monitoring") if isinstance(config, dict) else None + existing = (mon or {}).get("install_id") if isinstance(mon, dict) else None + if isinstance(existing, str) and existing.strip(): + return existing + + minted = str(uuid.uuid4()) + try: + from hermes_cli.config import load_config, save_config + + fresh = load_config() + if isinstance(fresh, dict): + slot = fresh.setdefault("monitoring", {}) + if isinstance(slot, dict) and not str(slot.get("install_id") or "").strip(): + slot["install_id"] = minted + save_config(fresh) + except Exception: + logger.debug("install_id persist failed; using ephemeral id", exc_info=True) + # Keep the in-memory config consistent for this process either way. + if isinstance(config, dict): + config.setdefault("monitoring", {}) + if isinstance(config["monitoring"], dict): + config["monitoring"]["install_id"] = minted + return minted + + +__all__ = [ + "ensure_install_id", +] diff --git a/agent/monitoring/redaction.py b/agent/monitoring/redaction.py new file mode 100644 index 000000000000..312875901e6b --- /dev/null +++ b/agent/monitoring/redaction.py @@ -0,0 +1,71 @@ +"""Redaction applied to monitoring data before egress. + +One unconditional scrub, no modes, no knobs. Every string that leaves the +process passes through ``redact_for_export``: + + * Secrets first — wraps ``agent/redact.py::redact_sensitive_text(force=True)`` + plus bearer/token-shape patterns, and fails CLOSED: if the redactor cannot + run, the raw string is never emitted. + * PII second — e-mail addresses, phone numbers, and UUID-shaped identifiers + are rewritten to ``[email]`` / ``[phone]`` / ``[id]``. + +There is deliberately no setting to weaken this. The monitoring plane is +content-free by design: rendered log messages are not exported, and bounded +structured strings are still scrubbed as defense-in-depth. This redactor also +remains available for a future, explicitly gated redacted-message detail mode. +""" + +from __future__ import annotations + +import re +from typing import Optional + +# ── secret shapes (belt-and-suspenders on top of agent/redact.py) ─────────── +_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+\-/]+=*", re.IGNORECASE) +_TOKEN_RE = re.compile( + r"\b(xox[baprs]-[A-Za-z0-9-]+|sk-[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]{8,})\b" +) +_SECRET_LITERAL_RE = re.compile(r"\*{3,}") +_BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+\[[^\]]+\]", re.IGNORECASE) + +# ── PII shapes ─────────────────────────────────────────────────────────────── +_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") +# E.164-ish and common separators; conservative to avoid nuking code/IDs. +_PHONE_RE = re.compile( + r"(? str: + """Always-on secret redaction. force=True so user config can't disable it.""" + try: + from agent.redact import redact_sensitive_text + out = redact_sensitive_text(text, force=True) + except Exception: + # Fail CLOSED: if the redactor can't run, do not emit the raw string. + return "[redaction-unavailable]" + out = _BEARER_RE.sub("[redacted]", out) + out = _TOKEN_RE.sub("[redacted]", out) + out = _SECRET_LITERAL_RE.sub("[redacted]", out) + out = _BEARER_RESIDUE_RE.sub("[redacted]", out) + return out + + +def redact_for_export(text: Optional[str]) -> Optional[str]: + """Scrub a string for egress: secrets, then PII. Unconditional.""" + if text is None: + return None + out = _secret_redact(str(text)) + out = _EMAIL_RE.sub("[email]", out) + out = _UUID_RE.sub("[id]", out) + out = _PHONE_RE.sub("[phone]", out) + return out + + +__all__ = [ + "redact_for_export", +] diff --git a/agent/nous_rate_guard.py b/agent/nous_rate_guard.py index 415d367ca17b..0234eef2ea23 100644 --- a/agent/nous_rate_guard.py +++ b/agent/nous_rate_guard.py @@ -117,7 +117,7 @@ def record_nous_rate_limit( # Atomic write: write to temp file + rename fd, tmp_path = tempfile.mkstemp(dir=state_dir, suffix=".tmp") try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(state, f) atomic_replace(tmp_path, path) except Exception: diff --git a/agent/outbound_webhooks.py b/agent/outbound_webhooks.py new file mode 100644 index 000000000000..f437b809f353 --- /dev/null +++ b/agent/outbound_webhooks.py @@ -0,0 +1,569 @@ +""" +Outbound webhook notifications. + +Reads the ``hooks.outbound:`` list from ``config.yaml`` and registers +notify-only callbacks on the existing plugin hook manager, so every +``invoke_hook()`` site can push lifecycle events to external HTTP +endpoints — CI systems, dashboards, other agents — with zero changes to +call sites and zero polling on the receiving end. + +This is the outbound mirror of the inbound webhook platform +(``gateway/platforms/webhook.py``): inbound wakes Hermes when the world +changes; outbound tells the world when Hermes does something. + +Design notes +------------ +* Delivery is fire-and-forget through a bounded in-process queue and a + single daemon worker thread. ``invoke_hook()`` runs inside the agent + loop, so callbacks must never block on network I/O — they serialize, + enqueue, and return ``None`` immediately. Outbound targets can never + block a tool call, inject context, or otherwise influence agent flow. +* Payloads are signed with HMAC-SHA256 (GitHub-style + ``X-Hermes-Signature-256: sha256=`` over the raw body) when + a secret is configured. Receivers verify exactly like they verify + GitHub webhooks. +* No consent prompt: unlike shell hooks, an outbound target executes no + code on this machine — it POSTs JSON to a URL the user themselves put + in config. ``HERMES_SAFE_MODE=1`` still skips registration, matching + plugins / MCP / shell hooks. +* Registration is idempotent — safe to invoke from both the CLI entry + point and the gateway entry point. + +Config schema (``~/.hermes/config.yaml``):: + + hooks: + outbound: + - url: https://ci.example.com/hermes-events + events: [on_session_end, subagent_stop] + # secret literal (discouraged) or env var name (preferred): + secret_env: HERMES_OUTBOUND_WEBHOOK_SECRET + # optional regex, honored for pre/post_tool_call only: + matcher: "terminal|delegate_task" + timeout: 10 # per-attempt seconds, clamped to [1, 60] + name: ci-notify # optional label for logs / `hermes hooks list` + +Wire format (POST body):: + + { + "hook_event_name": "on_session_end", + "tool_name": null, + "tool_input": null, + "session_id": "sess_abc123", + "cwd": "/home/user/project", + "extra": {...}, # event-specific kwargs + "delivery_id": "3f2c...", # uuid4, unique per POST + "timestamp": "2026-07-22T14:00:00Z" + } + +Headers:: + + Content-Type: application/json + User-Agent: Hermes-Agent-Outbound-Webhook + X-Hermes-Event: + X-Hermes-Delivery: + X-Hermes-Signature-256: sha256= # only when secret set +""" + +from __future__ import annotations + +import atexit +import hashlib +import hmac +import json +import logging +import os +import queue +import re +import threading +import time +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple +from urllib import error as urlerror +from urllib import request as urlrequest + +logger = logging.getLogger(__name__) + +DEFAULT_TIMEOUT_SECONDS = 10 +MAX_TIMEOUT_SECONDS = 60 +MAX_DELIVERY_ATTEMPTS = 2 +RETRY_BACKOFF_SECONDS = 1.0 +QUEUE_MAX_SIZE = 256 + +# Events whose ``matcher`` field is honored (mirrors shell hooks). +_TOOL_SCOPED_EVENTS = {"pre_tool_call", "post_tool_call"} + +# kwargs promoted to top-level payload keys (mirrors shell hooks wire). +_TOP_LEVEL_PAYLOAD_KEYS = {"tool_name", "args", "session_id", "parent_session_id"} + +# (event, url) pairs already wired to the plugin manager in this process. +_registered: Set[Tuple[str, str]] = set() +_registered_lock = threading.Lock() + +_delivery_queue: "queue.Queue[Optional[Dict[str, Any]]]" = queue.Queue( + maxsize=QUEUE_MAX_SIZE +) +_worker_lock = threading.Lock() +_worker: Optional[threading.Thread] = None + + +@dataclass +class WebhookTarget: + """Parsed and validated representation of one ``hooks.outbound`` entry.""" + + url: str + events: List[str] + name: str = "" + secret: Optional[str] = None + matcher: Optional[str] = None + timeout: int = DEFAULT_TIMEOUT_SECONDS + compiled_matcher: Optional[re.Pattern] = field(default=None, repr=False) + + def __post_init__(self) -> None: + if isinstance(self.matcher, str): + stripped = self.matcher.strip() + self.matcher = stripped if stripped else None + if self.matcher: + try: + self.compiled_matcher = re.compile(self.matcher) + except re.error as exc: + logger.warning( + "outbound webhook matcher %r is invalid (%s) — treating " + "as literal equality", self.matcher, exc, + ) + self.compiled_matcher = None + + @property + def label(self) -> str: + return self.name or self.url + + def matches_tool(self, tool_name: Optional[str]) -> bool: + if not self.matcher: + return True + if tool_name is None: + return False + if self.compiled_matcher is not None: + return self.compiled_matcher.fullmatch(tool_name) is not None + return tool_name == self.matcher + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def register_from_config(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: + """Register every configured outbound webhook on the plugin manager. + + ``cfg`` is the full parsed config dict. Missing, empty, or malformed + ``hooks.outbound`` is treated as zero targets — config parsing never + raises, because a broken webhook entry must not crash the agent. + + Returns the targets that ended up wired (deduplicated across repeat + calls, so the CLI and gateway can both invoke this safely). + """ + if not isinstance(cfg, dict): + return [] + + from utils import env_var_enabled + + if env_var_enabled("HERMES_SAFE_MODE"): + logger.info("HERMES_SAFE_MODE=1 — outbound webhook registration skipped") + return [] + + hooks_cfg = cfg.get("hooks") + targets = _parse_outbound_block( + hooks_cfg.get("outbound") if isinstance(hooks_cfg, dict) else None + ) + if not targets: + return [] + + from hermes_cli.plugins import get_plugin_manager + + manager = get_plugin_manager() + + registered: List[WebhookTarget] = [] + with _registered_lock: + for target in targets: + wired_any = False + for event in target.events: + key = (event, target.url) + if key in _registered: + continue + manager._hooks.setdefault(event, []).append( + _make_callback(event, target) + ) + _registered.add(key) + wired_any = True + logger.info( + "outbound webhook registered: %s -> %s (matcher=%s, " + "timeout=%ds)", + event, target.label, target.matcher, target.timeout, + ) + if wired_any: + registered.append(target) + + return registered + + +def iter_configured_targets(cfg: Optional[Dict[str, Any]]) -> List[WebhookTarget]: + """Parse ``hooks.outbound`` without registering anything. + Used by ``hermes hooks list``.""" + if not isinstance(cfg, dict): + return [] + hooks_cfg = cfg.get("hooks") + return _parse_outbound_block( + hooks_cfg.get("outbound") if isinstance(hooks_cfg, dict) else None + ) + + +def flush(timeout: float = 5.0) -> bool: + """Block until all queued deliveries are done (or *timeout* elapses). + Returns ``True`` when the queue fully drained. Test/shutdown helper.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with _delivery_queue.all_tasks_done: + if _delivery_queue.unfinished_tasks == 0: + return True + time.sleep(0.02) + with _delivery_queue.all_tasks_done: + return _delivery_queue.unfinished_tasks == 0 + + +def reset_for_tests() -> None: + """Clear the idempotence set and drain the queue. Test-only helper.""" + with _registered_lock: + _registered.clear() + try: + while True: + _delivery_queue.get_nowait() + _delivery_queue.task_done() + except queue.Empty: + pass + + +# --------------------------------------------------------------------------- +# Config parsing +# --------------------------------------------------------------------------- + +def _parse_outbound_block(raw: Any) -> List[WebhookTarget]: + if raw is None: + return [] + if not isinstance(raw, list): + logger.warning( + "hooks.outbound must be a list of webhook targets; got %s", + type(raw).__name__, + ) + return [] + + targets: List[WebhookTarget] = [] + for i, entry in enumerate(raw): + target = _parse_single_target(i, entry) + if target is not None: + targets.append(target) + return targets + + +def _parse_single_target(index: int, raw: Any) -> Optional[WebhookTarget]: + from hermes_cli.plugins import VALID_HOOKS + + if not isinstance(raw, dict): + logger.warning( + "hooks.outbound[%d] must be a mapping with 'url' and 'events' " + "keys; got %s", index, type(raw).__name__, + ) + return None + + url = raw.get("url") + if not isinstance(url, str) or not url.strip(): + logger.warning("hooks.outbound[%d] is missing a non-empty 'url'", index) + return None + url = url.strip() + if not url.lower().startswith(("http://", "https://")): + logger.warning( + "hooks.outbound[%d].url must be http(s); got %r — skipped", + index, url, + ) + return None + if url.lower().startswith("http://"): + logger.warning( + "hooks.outbound[%d].url uses plain http:// — payloads (including " + "tool inputs) travel unencrypted. Prefer https.", index, + ) + + events_raw = raw.get("events") + if not isinstance(events_raw, list) or not events_raw: + logger.warning( + "hooks.outbound[%d] needs a non-empty 'events' list (valid: %s)", + index, ", ".join(sorted(VALID_HOOKS)), + ) + return None + events: List[str] = [] + for ev in events_raw: + if ev in VALID_HOOKS: + events.append(ev) + else: + logger.warning( + "hooks.outbound[%d]: unknown event %r ignored (valid: %s)", + index, ev, ", ".join(sorted(VALID_HOOKS)), + ) + if not events: + logger.warning( + "hooks.outbound[%d] has no valid events — skipped", index, + ) + return None + + matcher = raw.get("matcher") + if matcher is not None and not isinstance(matcher, str): + logger.warning( + "hooks.outbound[%d].matcher must be a string regex; ignoring", + index, + ) + matcher = None + if matcher is not None and not any(e in _TOOL_SCOPED_EVENTS for e in events): + logger.warning( + "hooks.outbound[%d].matcher=%r will be ignored — matcher is only " + "honored for pre_tool_call / post_tool_call.", index, matcher, + ) + matcher = None + + timeout_raw = raw.get("timeout", DEFAULT_TIMEOUT_SECONDS) + try: + timeout = int(timeout_raw) + except (TypeError, ValueError): + logger.warning( + "hooks.outbound[%d].timeout must be an int (got %r); using " + "default %ds", index, timeout_raw, DEFAULT_TIMEOUT_SECONDS, + ) + timeout = DEFAULT_TIMEOUT_SECONDS + timeout = max(1, min(timeout, MAX_TIMEOUT_SECONDS)) + + secret = _resolve_secret(index, raw) + + name = raw.get("name") + if not isinstance(name, str): + name = "" + + return WebhookTarget( + url=url, + events=events, + name=name.strip(), + secret=secret, + matcher=matcher, + timeout=timeout, + ) + + +def _resolve_secret(index: int, raw: Dict[str, Any]) -> Optional[str]: + """``secret_env`` (env var name, preferred) wins over inline ``secret``.""" + secret_env = raw.get("secret_env") + if isinstance(secret_env, str) and secret_env.strip(): + value = os.environ.get(secret_env.strip(), "") + if value: + return value + logger.warning( + "hooks.outbound[%d].secret_env=%r is not set in the environment " + "— deliveries will be UNSIGNED", index, secret_env.strip(), + ) + return None + secret = raw.get("secret") + if isinstance(secret, str) and secret: + return secret + return None + + +# --------------------------------------------------------------------------- +# Callback + delivery +# --------------------------------------------------------------------------- + +def _make_callback(event: str, target: WebhookTarget): + """Build the notify-only closure ``invoke_hook()`` calls per firing.""" + + def _callback(**kwargs: Any) -> None: + if event in _TOOL_SCOPED_EVENTS: + if not target.matches_tool(kwargs.get("tool_name")): + return None + delivery_id = uuid.uuid4().hex + try: + body = _serialize_payload(event, kwargs, delivery_id) + except Exception: # defensive — a bad payload must not hurt the loop + logger.warning( + "outbound webhook payload serialization failed (event=%s " + "target=%s)", event, target.label, exc_info=True, + ) + return None + _enqueue(_build_delivery(event, target, body, delivery_id)) + return None + + _callback.__name__ = f"outbound_webhook[{event}:{target.label}]" + _callback.__qualname__ = _callback.__name__ + return _callback + + +def _serialize_payload( + event: str, kwargs: Dict[str, Any], delivery_id: str, +) -> bytes: + """Render the POST body. Same top-level shape as shell hooks' stdin + (documented in :mod:`agent.shell_hooks`), plus delivery metadata. + + ``delivery_id`` is shared with the ``X-Hermes-Delivery`` header so + receivers can dedupe on either — and since it (plus ``timestamp``) + lives inside the HMAC-signed body, it doubles as replay protection. + """ + extras = {k: v for k, v in kwargs.items() if k not in _TOP_LEVEL_PAYLOAD_KEYS} + try: + cwd = str(Path.cwd()) + except OSError: + cwd = "" + payload = { + "hook_event_name": event, + "tool_name": kwargs.get("tool_name"), + "tool_input": kwargs.get("args") if isinstance(kwargs.get("args"), dict) else None, + "session_id": kwargs.get("session_id") or kwargs.get("parent_session_id") or "", + "cwd": cwd, + "extra": extras, + "delivery_id": delivery_id, + "timestamp": datetime.now(tz=timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + } + return json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8") + + +def _build_delivery( + event: str, target: WebhookTarget, body: bytes, delivery_id: str, +) -> Dict[str, Any]: + headers = { + "Content-Type": "application/json", + "User-Agent": "Hermes-Agent-Outbound-Webhook", + "X-Hermes-Event": event, + "X-Hermes-Delivery": delivery_id, + } + if target.secret: + digest = hmac.new( + target.secret.encode("utf-8"), body, hashlib.sha256 + ).hexdigest() + headers["X-Hermes-Signature-256"] = f"sha256={digest}" + return { + "url": target.url, + "label": target.label, + "event": event, + "body": body, + "headers": headers, + "timeout": target.timeout, + } + + +def _enqueue(delivery: Dict[str, Any]) -> None: + _ensure_worker() + try: + _delivery_queue.put_nowait(delivery) + except queue.Full: + logger.warning( + "outbound webhook queue full (%d pending) — dropping %s event " + "for %s", QUEUE_MAX_SIZE, delivery["event"], delivery["label"], + ) + + +def _ensure_worker() -> None: + global _worker + if _worker is not None and _worker.is_alive(): + return + with _worker_lock: + if _worker is not None and _worker.is_alive(): + return + _worker = threading.Thread( + target=_worker_loop, name="outbound-webhooks", daemon=True, + ) + _worker.start() + # The worker is a daemon thread, so a short-lived process (a `-q` + # CLI run, a cron session) can exit right after enqueuing the + # final events — silently dropping on_session_end, the headline + # use case. Drain the queue at interpreter shutdown, bounded so + # a dead endpoint can only delay exit, never hang it. + atexit.register(flush, timeout=5.0) + + +def _worker_loop() -> None: + while True: + delivery = _delivery_queue.get() + try: + if delivery is not None: + _deliver(delivery) + except Exception: # pragma: no cover — defensive + logger.warning( + "outbound webhook delivery crashed (target=%s)", + delivery.get("label") if isinstance(delivery, dict) else "?", + exc_info=True, + ) + finally: + _delivery_queue.task_done() + + +class _NoRedirectHandler(urlrequest.HTTPRedirectHandler): + """Refuse to follow redirects. + + urllib's default handler converts a redirected POST into a body-less + GET — the signed payload would be silently dropped and the headers + re-sent to a location the user never configured. Treat any 3xx as a + delivery failure instead (surfaced as HTTPError by returning None). + """ + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: D102 + return None + + +_opener = urlrequest.build_opener(_NoRedirectHandler) + + +def _deliver(delivery: Dict[str, Any]) -> None: + """POST with bounded retries. Retries on connection errors and 5xx; + 4xx is the receiver telling us the request itself is wrong — no retry. + 3xx redirects are never followed (misconfiguration — fix the URL).""" + last_error = "" + for attempt in range(1, MAX_DELIVERY_ATTEMPTS + 1): + req = urlrequest.Request( + delivery["url"], + data=delivery["body"], + headers=delivery["headers"], + method="POST", + ) + try: + with _opener.open(req, timeout=delivery["timeout"]) as resp: + status = getattr(resp, "status", 200) + if 200 <= status < 300: + logger.debug( + "outbound webhook delivered: %s -> %s (HTTP %d)", + delivery["event"], delivery["label"], status, + ) + return + last_error = f"HTTP {status}" + except urlerror.HTTPError as exc: + last_error = f"HTTP {exc.code}" + if 300 <= exc.code < 400: + logger.warning( + "outbound webhook target redirected (event=%s target=%s): " + "%s -> %s — redirects are not followed; update the " + "configured url", delivery["event"], delivery["label"], + last_error, exc.headers.get("Location", "?"), + ) + return + if 400 <= exc.code < 500: + logger.warning( + "outbound webhook rejected (event=%s target=%s): %s — " + "not retrying", delivery["event"], delivery["label"], + last_error, + ) + return + except Exception as exc: + last_error = str(exc) or type(exc).__name__ + + if attempt < MAX_DELIVERY_ATTEMPTS: + time.sleep(RETRY_BACKOFF_SECONDS * attempt) + + logger.warning( + "outbound webhook delivery failed after %d attempt(s) (event=%s " + "target=%s): %s", + MAX_DELIVERY_ATTEMPTS, delivery["event"], delivery["label"], last_error, + ) diff --git a/agent/plugin_llm.py b/agent/plugin_llm.py index e9c2a869dd76..8fcd3364b1c2 100644 --- a/agent/plugin_llm.py +++ b/agent/plugin_llm.py @@ -210,8 +210,8 @@ def _resolve_trust_policy(plugin_id: str) -> _TrustPolicy: return _TrustPolicy(plugin_id="") try: - from hermes_cli.config import load_config - config = load_config() or {} + from hermes_cli.config import load_config_readonly + config = load_config_readonly() or {} except Exception: # pragma: no cover — config IO failure return _TrustPolicy(plugin_id=plugin_id) diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 845e4260ddbd..0f748f8b2bd2 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -19,13 +19,18 @@ from agent.runtime_cwd import resolve_agent_cwd from agent.skill_utils import ( EXCLUDED_SKILL_DIRS, + ORG_ACTIVE_MARKER, + ORG_MIRROR_DIR_NAME, + ORG_PROVENANCE_FILE, SKILL_SUPPORT_DIRS, extract_skill_conditions, extract_skill_description, get_all_skills_dirs, get_disabled_skill_names, iter_skill_index_files, + org_id_of_path, parse_frontmatter, + read_active_org_id, skill_matches_environment, skill_matches_platform, skill_matches_platform_list, @@ -567,16 +572,18 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str: "Background delivery is the DEFAULT and the co-work path, but it is " "the first rung, not the only one. Read each action's structured " "result and climb only when the driver tells you to:\n" - "- `effect: 'confirmed'` + `verified: true` — the driver read the " - "result back. Done.\n" + "- `effect: 'confirmed'` (or `verified: true`) — done, even if an " + "advisory escalation is also present. Never repeat successful input.\n" "- `effect: 'unverifiable'` — the input was delivered but the driver " - "can't confirm it. Re-capture and check the screenshot/tree yourself " - "before deciding it worked.\n" - "- `effect: 'suspected_noop'`, `code: 'background_unavailable'`, or an " - "`escalation.recommended` field — the action did NOT land. Follow " - "`escalation.recommended`:\n" + "can't confirm it. Get fresh state and check it before any retry; an " + "escalation recommendation does not override this rule.\n" + "- `effect: 'suspected_noop'` or a structured refusal such as " + "`code: 'background_unavailable'` — escalation is allowed. Follow " + "the recommended rung when present:\n" " - `'px'` → re-issue addressing the target by `coordinate=[x,y]` " "read off the screenshot instead of `element`.\n" + " - `'page'` → use the exact-bound typed browser page rung below " + "before native foreground escalation. Do not start a legacy page workflow.\n" " - `'foreground'` (or a pixel click still didn't land) → re-issue " "the SAME action with `delivery_mode='foreground'`. This briefly " "raises the window; it needs its own approval and is only appropriate " @@ -586,6 +593,21 @@ def computer_use_guidance(platform_name: Optional[str] = None) -> str: "as a prediction from the app being Electron/Chromium/GTK. Do not " "silently retry the same rung expecting a different result, and do " "not conclude 'cua-driver can't drive this app' — climb the ladder.\n\n" + "## Typed browser page rung\n" + "For `recommended='page'` or supported browser PAGE content, use the namespaced " + "`cua_browser_*` actions: bind with `cua_browser_state` using the exact " + "native `(pid, window_id)`, require `binding_quality='exact'` and " + "`mutation_allowed=true`, select its opaque `tab_id`, then take a " + "fresh semantic snapshot before using a current `ref`. After every " + "typed mutation, call `cua_browser_state` again before another action. " + "Input defaults to trusted; `input_route='dom_event'` is an explicit " + "downgrade, never an automatic retry. Use native capture/input for " + "browser chrome, OS permission prompts, native dialogs, and unsupported " + "targets. Browser setup is a separately approved action; attaching an " + "existing profile is enforced by cua-driver's immutable permission " + "mode: standard requires a certified protected host and fails closed " + "when Hermes has none; explicit Hermes YOLO uses a private unrestricted " + "daemon after the user's launch/session risk acceptance.\n\n" "## Background mode rules\n" "- Do NOT use `raise_window=true` on `focus_app` unless the user " "explicitly asked you to bring a window to front. Input routing to " @@ -959,7 +981,7 @@ def format_steer_marker(steer_text: str) -> str: # misleading — the agent should only see the machine it can actually touch. _REMOTE_TERMINAL_BACKENDS = frozenset({ "docker", "singularity", "modal", "daytona", "ssh", - "managed_modal", + "vercel_sandbox", "managed_modal", }) @@ -973,6 +995,7 @@ def format_steer_marker(steer_text: str) -> str: "modal": "a Modal sandbox (Linux)", "managed_modal": "a managed Modal sandbox (Linux)", "daytona": "a Daytona workspace (Linux)", + "vercel_sandbox": "a Vercel sandbox (Linux)", "ssh": "a remote host reached over SSH (likely Linux)", } @@ -1047,7 +1070,7 @@ def _probe_remote_backend(env_type: str) -> str | None: } container_config = None - if env_type in {"docker", "singularity", "modal", "daytona"}: + if env_type in {"docker", "singularity", "modal", "daytona", "vercel_sandbox"}: container_config = { "container_cpu": config.get("container_cpu", 1), "container_memory": config.get("container_memory", 5120), @@ -1060,6 +1083,7 @@ def _probe_remote_backend(env_type: str) -> str | None: "docker_env": config.get("docker_env", {}), "docker_run_as_host_user": config.get("docker_run_as_host_user", False), "docker_extra_args": config.get("docker_extra_args", []), + "docker_shm_size": config.get("docker_shm_size", "1g"), "docker_persist_across_processes": config.get("docker_persist_across_processes", True), "docker_orphan_reaper": config.get("docker_orphan_reaper", True), } @@ -1137,7 +1161,7 @@ def build_environment_hints() -> str: and a Windows-only note that `terminal` shells out to bash, not PowerShell). - For **remote / sandbox** terminal backends (docker, singularity, - modal, daytona, ssh): host info is **suppressed** + modal, daytona, ssh, vercel_sandbox): host info is **suppressed** because the agent's tools can't touch the host — only the backend matters. A live probe inside the backend reports its OS, user, $HOME, and cwd. Falls back to a static summary if the probe fails. @@ -1224,10 +1248,10 @@ def build_environment_hints() -> str: extra = (os.getenv("HERMES_ENVIRONMENT_HINT") or "").strip() if not extra: try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly extra = str( - (load_config().get("agent", {}) or {}).get("environment_hint", "") + (load_config_readonly().get("agent", {}) or {}).get("environment_hint", "") ).strip() except Exception as e: logger.debug("Could not read agent.environment_hint from config: %s", e) @@ -1278,9 +1302,9 @@ def _get_context_file_max_chars(context_length: Optional[int] = None) -> int: 3. ``CONTEXT_FILE_MAX_CHARS`` (20K) as the upstream-compatible fallback. """ try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - val = load_config().get("context_file_max_chars") + val = load_config_readonly().get("context_file_max_chars") if isinstance(val, (int, float)) and val > 0: return int(val) except Exception as e: @@ -1323,7 +1347,9 @@ def drain_truncation_warnings() -> list: _SKILLS_PROMPT_CACHE_MAX = 8 _SKILLS_PROMPT_CACHE: OrderedDict[tuple, str] = OrderedDict() _SKILLS_PROMPT_CACHE_LOCK = threading.Lock() -_SKILLS_SNAPSHOT_VERSION = 1 +# v2: entries gained org provenance fields (org_id/org_author/rel_dir) for M2 +# org-shared skills; older snapshots are discarded and rebuilt. +_SKILLS_SNAPSHOT_VERSION = 2 def _skills_prompt_snapshot_path() -> Path: @@ -1342,13 +1368,32 @@ def clear_skills_system_prompt_cache(*, clear_snapshot: bool = False) -> None: def _build_skills_manifest(skills_dir: Path) -> dict[str, list[int]]: - """Build an mtime/size manifest of all SKILL.md and DESCRIPTION.md files.""" + """Build an mtime/size manifest of all SKILL.md and DESCRIPTION.md files. + + Org mirrors (M2): only the ACTIVE org's mirror participates, and the + ``.active_org`` marker itself is included — so switching/leaving an org + invalidates the snapshot even when no SKILL.md changed. + """ manifest: dict[str, list[int]] = {} skills_dir_str = str(skills_dir) base = os.path.join(skills_dir_str, "") prefix_len = len(base) + active_org = read_active_org_id(skills_dir) + org_root = os.path.join(skills_dir_str, ORG_MIRROR_DIR_NAME) + marker_path = os.path.join(org_root, ORG_ACTIVE_MARKER) + try: + st = os.stat(marker_path) + manifest[ORG_MIRROR_DIR_NAME + "/" + ORG_ACTIVE_MARKER] = [ + int(st.st_mtime), int(st.st_size), + ] + except OSError: + pass for root, dirs, files in os.walk(skills_dir_str, followlinks=True): has_skill_md = "SKILL.md" in files + if root == skills_dir_str and ORG_MIRROR_DIR_NAME in dirs and active_org is None: + dirs.remove(ORG_MIRROR_DIR_NAME) + elif root == org_root: + dirs[:] = [d for d in dirs if d == active_org] dirs[:] = [ d for d in dirs @@ -1413,6 +1458,15 @@ def _build_snapshot_entry( """Build a serialisable metadata dict for one skill.""" rel_path = skill_file.relative_to(skills_dir) parts = rel_path.parts + + # M2 org mirror: strip the `_org//` prefix so category/name derive + # from the path WITHIN the mirror (same shape the org tree was built + # from), and record provenance for labeling + fail-loud collisions. + org_id: str | None = None + if len(parts) >= 3 and parts[0] == ORG_MIRROR_DIR_NAME: + org_id = parts[1] + parts = parts[2:] + if len(parts) >= 2: skill_name = parts[-2] category = "/".join(parts[:-2]) if len(parts) > 2 else parts[0] @@ -1424,7 +1478,7 @@ def _build_snapshot_entry( if isinstance(platforms, str): platforms = [platforms] - return { + entry = { "skill_name": skill_name, "category": category, "frontmatter_name": str(frontmatter.get("name", skill_name)), @@ -1432,6 +1486,22 @@ def _build_snapshot_entry( "platforms": [str(p).strip() for p in platforms if str(p).strip()], "conditions": extract_skill_conditions(frontmatter), } + if org_id: + entry["org_id"] = org_id + # Author from the pull-time provenance sidecar (token-verified at + # push by the plane's author_mismatch guard). Best-effort. + try: + import json as _json + + prov_path = ( + skills_dir / ORG_MIRROR_DIR_NAME / org_id / ORG_PROVENANCE_FILE + ) + prov = _json.loads(prov_path.read_text(encoding="utf-8")) + device = str(prov.get("author_device") or "") + entry["org_author"] = device or str(prov.get("author_user_id") or "") + except Exception: + entry["org_author"] = "" + return entry # ========================================================================= @@ -1567,6 +1637,10 @@ def build_skills_system_prompt( skills_by_category: dict[str, list[tuple[str, str]]] = {} category_descriptions: dict[str, str] = {} + # Unified visible-entry list (both paths) so the org labeling + + # fail-loud collision pass below runs identically for snapshot and scan. + visible_entries: list[dict] = [] + skill_entries: list[dict] = [] if snapshot is not None: # Fast path: use pre-parsed metadata from disk @@ -1574,7 +1648,6 @@ def build_skills_system_prompt( if not isinstance(entry, dict): continue skill_name = entry.get("skill_name") or "" - category = entry.get("category") or "general" frontmatter_name = entry.get("frontmatter_name") or skill_name platforms = entry.get("platforms") or [] if not skill_matches_platform_list(platforms): @@ -1587,16 +1660,13 @@ def build_skills_system_prompt( available_toolsets, ): continue - skills_by_category.setdefault(category, []).append( - (frontmatter_name, entry.get("description", "")) - ) + visible_entries.append(entry) category_descriptions = { str(k): str(v) for k, v in (snapshot.get("category_descriptions") or {}).items() } else: # Cold path: full filesystem scan + write snapshot for next time - skill_entries: list[dict] = [] for skill_file in iter_skill_index_files(skills_dir, "SKILL.md"): is_compatible, frontmatter, desc = _parse_skill_file(skill_file) entry = _build_snapshot_entry(skill_file, skills_dir, frontmatter, desc) @@ -1612,10 +1682,38 @@ def build_skills_system_prompt( available_toolsets, ): continue - skills_by_category.setdefault(entry["category"], []).append( - (entry["frontmatter_name"], entry["description"]) - ) + visible_entries.append(entry) + + # ── M2 org labeling + FAIL-LOUD collisions ───────────────────────── + # An org skill lists with an explicit provenance tag. When a personal and + # an org skill share a name, NEITHER silently wins: both list qualified + # (personal keeps the bare name is the wrong default — silent divergence + # from the org set; org winning silently shadows the user's own work) — + # so both entries carry a [name collision] flag and skill_view refuses + # the ambiguous bare name (its existing multi-candidate guard). + name_owners: dict[str, set[str]] = {} + for entry in visible_entries: + fm = entry.get("frontmatter_name") or entry.get("skill_name") or "" + kind = "org" if entry.get("org_id") else "personal" + name_owners.setdefault(fm, set()).add(kind) + for entry in visible_entries: + fm = entry.get("frontmatter_name") or entry.get("skill_name") or "" + desc = entry.get("description", "") + org_id = entry.get("org_id") + collided = len(name_owners.get(fm, set())) > 1 + if org_id: + author = entry.get("org_author") or "" + tag = f"[org-shared{': by ' + author if author else ''}]" + desc = f"{tag} {desc}".strip() + category = f"org:{org_id}" + else: + category = entry.get("category") or "general" + if collided: + desc = f"[name collision — also exists {'personally' if org_id else 'in your org'}; load via category path] {desc}".strip() + skills_by_category.setdefault(category, []).append((fm, desc)) + if snapshot is None: + # (continuation of the cold path below: category descriptions + write) # Read category-level DESCRIPTION.md files for desc_file in iter_skill_index_files(skills_dir, "DESCRIPTION.md"): try: diff --git a/agent/prompt_caching.py b/agent/prompt_caching.py index 9a2fdf4ccce6..a012f143f2c7 100644 --- a/agent/prompt_caching.py +++ b/agent/prompt_caching.py @@ -1,17 +1,37 @@ """Anthropic prompt caching strategy. -Single layout: ``system_and_3``. 4 cache_control breakpoints — system -prompt + last 3 non-system messages, all at the same TTL (5m or 1h). -Reduces input token costs by ~75% on multi-turn conversations within a -single session. +The default layout uses 4 cache_control breakpoints: the static system +prefix, the end of the system prompt, and the last 2 non-system messages. +When a static system prefix is unavailable, it falls back to one system +breakpoint plus the last 3 messages. All markers use the same TTL (5m or 1h). +This preserves intra-session caching while allowing new sessions to reuse the +stable system-prompt prefix. Pure functions -- no class state, no AIAgent dependency. """ import copy +from dataclasses import dataclass from typing import Any, Dict, List +@dataclass(frozen=True) +class PromptCachePlan: + """Request-local message and tool sections with their cache markers.""" + + messages: List[Dict[str, Any]] + tools: List[Dict[str, Any]] + + @property + def marker_count(self) -> int: + """Wire-visible cache markers in this plan (computed on demand). + + Only tests consume this; keeping it lazy avoids walking every + message part and tool schema on the per-request hot path. + """ + return _count_cache_markers(self.messages, self.tools) + + def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool = False) -> None: """Add cache_control to a single message, handling all format variations.""" role = msg.get("role", "") @@ -81,30 +101,283 @@ def _build_marker(ttl: str) -> Dict[str, str]: return marker +def _apply_system_cache_markers( + message: dict, + cache_marker: dict, + static_system_prefix: str | None, + *, + native_anthropic: bool, + mark_suffix: bool = True, + fallback_to_whole: bool = True, +) -> int: + """Mark the static system prefix (and optionally the full prompt). + + The system prompt remains one stored string. Splitting it only in the + outgoing request keeps session persistence and non-Anthropic transports + unchanged while making the stable prefix independently cacheable. + + ``mark_suffix=False`` is the tool-cache-plan layout: only the static + prefix carries a marker, the volatile suffix rides unmarked (its + breakpoint budget is spent on the tools array instead). + + ``fallback_to_whole=False`` skips marking entirely when the prefix + split is not possible (no prefix, mismatched prefix, non-string + content) instead of marking the whole message. + + When the prompt IS exactly the static prefix (empty suffix), the whole + message is marked as a single block — never a two-part split with an + empty text block, which Anthropic rejects. + + Returns the number of markers applied (0, 1, or 2). + """ + content = message.get("content") + if ( + isinstance(static_system_prefix, str) + and static_system_prefix + and isinstance(content, str) + and content.startswith(static_system_prefix) + ): + suffix = content[len(static_system_prefix):] + if suffix: + suffix_part: dict = {"type": "text", "text": suffix} + if mark_suffix: + suffix_part["cache_control"] = cache_marker + message["content"] = [ + { + "type": "text", + "text": static_system_prefix, + "cache_control": cache_marker, + }, + suffix_part, + ] + return 2 if mark_suffix else 1 + # Empty suffix: the stored prompt IS the static prefix. Mark it as + # one whole block — a [marked-prefix, ""] split would put an empty + # text block on the wire (HTTP 400 on native Anthropic). + _apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic) + return 1 + + if not fallback_to_whole: + return 0 + _apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic) + return 1 + + +def strip_anthropic_cache_control( + api_messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Remove ``cache_control`` markers and undo decoration-produced list shapes. + + Used before re-applying decoration after a mid-turn provider failover so + the mutated, undecorated shape (image shrink / ASCII cleanup / etc.) is + preserved while markers match the *new* provider's cache policy (#72626). + + Flattening back to a plain string is restricted to the exact shapes + :func:`apply_anthropic_cache_control` produces from string content — + a single ``{"type": "text"}`` part, or the two-part ``[static, volatile]`` + system split — so the ``""``-join is provably byte-exact. Organic + multi-part text (merged user turns, imported transcripts) and parts + carrying extra keys (``citations`` etc.) keep their structure; only + per-part markers are removed. Marker removal is copy-on-write on the + part dicts: content parts may alias the persistent conversation history + (the per-call copy is shallow), and stripping must never rewrite the + stored transcript. + + Mutates the top-level message dicts of ``api_messages`` in place and + returns the same list. + """ + for msg in api_messages: + if not isinstance(msg, dict): + continue + msg.pop("cache_control", None) + content = msg.get("content") + if not isinstance(content, list): + continue + if any(isinstance(part, dict) and "cache_control" in part for part in content): + content = [ + {k: v for k, v in part.items() if k != "cache_control"} + if isinstance(part, dict) and "cache_control" in part + else part + for part in content + ] + msg["content"] = content + decoration_shape = content and all( + isinstance(part, dict) + and part.get("type", "text") == "text" + and isinstance(part.get("text"), str) + and set(part.keys()) <= {"type", "text"} + for part in content + ) and ( + len(content) == 1 + or (msg.get("role") == "system" and len(content) == 2) + ) + if decoration_shape: + msg["content"] = "".join(part["text"] for part in content) + return api_messages + + +def strip_anthropic_tool_cache_control(tools: List[Dict[str, Any]] | None) -> List[Dict[str, Any]]: + """Return copied tools without request-local Anthropic cache markers.""" + cleaned = copy.deepcopy(tools or []) + for tool in cleaned: + if isinstance(tool, dict): + tool.pop("cache_control", None) + return cleaned + + +def _count_cache_markers(messages: List[Dict[str, Any]], tools: List[Dict[str, Any]]) -> int: + """Count the wire-visible cache markers in a request-local plan.""" + count = sum( + 1 + for message in messages + if isinstance(message, dict) and "cache_control" in message + ) + count += sum( + 1 + for message in messages + if isinstance(message, dict) and isinstance(message.get("content"), list) + for part in message["content"] + if isinstance(part, dict) and "cache_control" in part + ) + return count + sum( + 1 for tool in tools if isinstance(tool, dict) and "cache_control" in tool + ) + + +def _completed_transaction_endpoint_indexes( + messages: List[Dict[str, Any]], *, native_anthropic: bool, +) -> List[int]: + """Select legal ends of completed tool runs and ordinary turns.""" + endpoints: List[int] = [] + index = 0 + while index < len(messages): + message = messages[index] + if not isinstance(message, dict) or message.get("role") == "system": + index += 1 + continue + + if message.get("role") == "assistant" and message.get("tool_calls"): + result_start = index + 1 + result_end = result_start + while result_end < len(messages): + result = messages[result_end] + if not isinstance(result, dict) or result.get("role") != "tool": + break + result_end += 1 + if result_end > result_start: + endpoint = result_end - 1 + if _can_carry_marker(messages[endpoint], native_anthropic): + endpoints.append(endpoint) + index = result_end + continue + + if message.get("role") == "tool": + while index < len(messages): + result = messages[index] + if not isinstance(result, dict) or result.get("role") != "tool": + break + index += 1 + continue + + if message.get("role") == "user" and index + 1 < len(messages): + index += 1 + continue + + if ( + message.get("role") == "assistant" + and message.get("content") in (None, "") + ): + index += 1 + continue + + if _can_carry_marker(message, native_anthropic): + endpoints.append(index) + index += 1 + return endpoints + + +def build_prompt_cache_plan( + api_messages: List[Dict[str, Any]], + tools: List[Dict[str, Any]] | None, + *, + cache_ttl: str = "5m", + native_anthropic: bool = False, + static_system_prefix: str | None = None, + direct_native_tool_cache: bool = False, +) -> PromptCachePlan: + """Build isolated cache sections for one resolved request destination.""" + messages = copy.deepcopy(api_messages or []) + strip_anthropic_cache_control(messages) + planned_tools = strip_anthropic_tool_cache_control(tools) + + if not direct_native_tool_cache or not planned_tools: + planned_messages = apply_anthropic_cache_control( + messages, + cache_ttl=cache_ttl, + native_anthropic=native_anthropic, + static_system_prefix=static_system_prefix, + ) + return PromptCachePlan(messages=planned_messages, tools=planned_tools) + + marker = _build_marker(cache_ttl) + if ( + messages + and isinstance(messages[0], dict) + and messages[0].get("role") == "system" + ): + # Tool-cache layout: only the static prefix carries a system-side + # marker; the volatile suffix's budget is spent on the tools array. + _apply_system_cache_markers( + messages[0], + marker, + static_system_prefix, + native_anthropic=True, + mark_suffix=False, + fallback_to_whole=False, + ) + planned_tools[-1]["cache_control"] = dict(marker) + for endpoint in _completed_transaction_endpoint_indexes( + messages, + native_anthropic=True, + )[-2:]: + _apply_cache_marker(messages[endpoint], marker, native_anthropic=True) + + return PromptCachePlan(messages=messages, tools=planned_tools) + + def apply_anthropic_cache_control( api_messages: List[Dict[str, Any]], cache_ttl: str = "5m", native_anthropic: bool = False, + static_system_prefix: str | None = None, ) -> List[Dict[str, Any]]: - """Apply system_and_3 caching strategy to messages for Anthropic models. + """Apply Anthropic cache-control markers to API messages. - Places up to 4 cache_control breakpoints: system prompt + last 3 non-system - messages, all at the same TTL. + When ``static_system_prefix`` exactly matches the beginning of a string + system prompt, it receives an early marker and the full system prompt gets + a trailing marker. The remaining two markers target the latest cacheable + non-system messages. Without that prefix, the legacy system-and-3 layout + is retained. Returns: - Deep copy of messages with cache_control breakpoints injected. + Shallow copy of message list with selective deep copies of modified messages. """ - messages = copy.deepcopy(api_messages) - if not messages: - return messages + if not api_messages: + return api_messages + messages = list(api_messages) marker = _build_marker(cache_ttl) breakpoints_used = 0 if messages[0].get("role") == "system": - _apply_cache_marker(messages[0], marker, native_anthropic=native_anthropic) - breakpoints_used += 1 + messages[0] = copy.deepcopy(messages[0]) + breakpoints_used = _apply_system_cache_markers( + messages[0], + marker, + static_system_prefix, + native_anthropic=native_anthropic, + ) remaining = 4 - breakpoints_used non_sys = [ @@ -114,6 +387,7 @@ def apply_anthropic_cache_control( and _can_carry_marker(messages[i], native_anthropic=native_anthropic) ] for idx in non_sys[-remaining:]: + messages[idx] = copy.deepcopy(messages[idx]) _apply_cache_marker(messages[idx], marker, native_anthropic=native_anthropic) return messages diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index c9bf00988e5e..277cd0186549 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -12,7 +12,10 @@ ironsh). It sits between the sandbox and the internet, enforces a default-deny allowlist on outbound hosts, and *swaps proxy tokens for real credentials* on the way out. The sandbox only ever holds opaque proxy tokens — leaking -them is useless, since they only work from behind the proxy. +them is useless, since they only work behind the configured trusted proxy +boundary (the CA private key and proxy endpoint integrity are part of that +boundary: if traffic can be redirected to attacker-controlled proxy +infrastructure, the guarantee no longer holds). Design summary -------------- @@ -37,8 +40,10 @@ at proxy startup instead. * The proxy runs as a managed subprocess (``hermes egress start``), pidfile - at ``/proxy/iron-proxy.pid``, structured audit log at - ``/proxy/audit.log``. + at ``/proxy/iron-proxy.pid``. Daemon output (including + per-request records on v0.39) goes to ``/proxy/iron-proxy.log``; + ``audit.log`` is pre-created but reserved for a future pin that supports + ``log.audit_path``. * Failures (binary missing, port collision, bad config) emit a one-line warning and do *not* block agent startup. The Docker backend refuses to @@ -53,6 +58,7 @@ from __future__ import annotations import hashlib +import ipaddress import json import logging import os @@ -63,6 +69,7 @@ import subprocess import tarfile import tempfile +import threading import time import urllib.error import urllib.request @@ -86,12 +93,36 @@ f"https://github.com/ironsh/iron-proxy/releases/download/v{_IRON_PROXY_VERSION}" ) _IRON_PROXY_CHECKSUM_NAME = "checksums.txt" +# Detached signature for checksums.txt + the signing public key, both shipped on +# the release. Used for optional GPG verification of the release channel +# (maxpetrusenko P1): SHA-256 only protects the archive if checksums.txt itself +# came from an uncompromised channel; verifying its signature closes that gap. +_IRON_PROXY_CHECKSUM_SIG_NAME = "checksums.txt.asc" +_IRON_PROXY_PUBKEY_NAME = "public-key.asc" # How long to wait for HTTP downloads and subprocess interactions, in seconds. _DOWNLOAD_TIMEOUT = 120 # binary is ~16MB _RUN_TIMEOUT = 30 _STARTUP_GRACE_SECONDS = 5 +# Management (operator) API. iron-proxy v0.39 ships an authenticated +# loopback HTTP endpoint (``management.listen`` + ``management.api_key_env``) +# whose ``POST /v1/reload`` re-reads proxy.yaml and atomically swaps the +# transform pipeline in-place — no restart, no dropped connections. We +# always enable it on generated configs: it binds loopback only and every +# request needs the bearer key below. ``hermes egress reload`` is the +# client. +# +# The key is minted at setup time, stored at +# ``/proxy/management.token`` (0600), and injected into the +# daemon's env under this name at start. v0.39 validates at startup that +# the named env var is non-empty when management.listen is set. +_MGMT_API_KEY_ENV = "HERMES_IRON_PROXY_MGMT_KEY" +# The management listener binds loopback at tunnel_port + 2 (tunnel_port +# is CONNECT/MITM, +1 is the plain-HTTP forward listener). +_MGMT_PORT_OFFSET = 2 +_MGMT_RELOAD_TIMEOUT = 15 + # Default listen ports. HTTPS_PROXY semantics use a single CONNECT tunnel, # so we expose only the tunnel listener for v1 — no need to put the sandbox # DNS at the iron-proxy IP. This greatly simplifies wiring. @@ -113,18 +144,8 @@ ) # Provider env-var name -> upstream host (or list of hosts) on which the -# sandbox-visible proxy token should be swapped for the real host-side -# credential. Most providers use ``Authorization: Bearer``; Google AI Studio -# / Gemini uses ``x-goog-api-key``. Providers with signatures/OAuth flows -# stay in ``_NON_BEARER_PROVIDERS`` until we have an explicit transform rule. +# Authorization Bearer token should be swapped. _BEARER_PROVIDERS: Dict[str, Tuple[str, ...]] = { - # Anthropic native uses x-api-key instead of Authorization, but iron-proxy's - # secrets transform can still swap a sandbox token for the host-side key. - # Support both common env names: Hermes/Gary's live runtime uses - # ANTHROPIC_TOKEN, while Anthropic SDK examples usually use - # ANTHROPIC_API_KEY. - "ANTHROPIC_API_KEY": ("api.anthropic.com",), - "ANTHROPIC_TOKEN": ("api.anthropic.com",), "OPENROUTER_API_KEY": ("openrouter.ai", "*.openrouter.ai"), "OPENAI_API_KEY": ("api.openai.com",), "GROQ_API_KEY": ("api.groq.com",), @@ -133,37 +154,71 @@ "MISTRAL_API_KEY": ("api.mistral.ai",), "XAI_API_KEY": ("api.x.ai",), "NOUS_API_KEY": ("inference.nousresearch.com",), - # Google AI Studio / Gemini native API. Hermes' native Gemini adapter sends - # the key in x-goog-api-key, not Authorization. build_proxy_config special - # cases the header below so sandboxes receive only opaque proxy tokens. - "GEMINI_API_KEY": ("generativelanguage.googleapis.com",), - "GOOGLE_API_KEY": ("generativelanguage.googleapis.com",), } -_PROVIDER_MATCH_HEADERS: Dict[str, Tuple[str, ...]] = { - "ANTHROPIC_API_KEY": ("x-api-key",), - "ANTHROPIC_TOKEN": ("Authorization",), - "GEMINI_API_KEY": ("x-goog-api-key",), - "GOOGLE_API_KEY": ("x-goog-api-key",), + +# Providers whose API authenticates with a NON-Authorization header. +# iron-proxy v0.39's ``secrets.replace.match_headers`` targets arbitrary +# header names (case-insensitive; confirmed by the iron-proxy author on +# PR #30179 and verified in the pinned v0.39.0 source — ``swapHeaders`` +# + ``parseHeaderMatchers``), so these are first-class swapped providers, +# not "uncovered". +# +# ``aliases`` are interchangeable env-var names for the SAME upstream +# credential (Hermes' auth.py keys Google on both GEMINI_API_KEY and +# GOOGLE_API_KEY). Aliased names MUST collapse into a single mapping: +# every rule carries ``require: true``, and two require-rules on the same +# host reject each other's requests (each rule whose own token isn't +# present returns ActionReject). The sandbox receives the minted token +# under the canonical name AND every alias so SDKs reading either work. +_HEADER_AUTH_PROVIDERS: Dict[str, Dict[str, Tuple[str, ...]]] = { + # Anthropic native: x-api-key. Authorization is also matched so an + # SDK sending the token as a Bearer (OAuth-style) still swaps. + "ANTHROPIC_API_KEY": { + "hosts": ("api.anthropic.com",), + "match_headers": ("x-api-key", "Authorization"), + "aliases": (), + }, + # Azure OpenAI: api-key header (AAD bearer flows use Authorization). + "AZURE_OPENAI_API_KEY": { + "hosts": ( + "*.openai.azure.com", + "*.cognitiveservices.azure.com", + "*.services.ai.azure.com", + ), + "match_headers": ("api-key", "Authorization"), + "aliases": (), + }, + # Google AI Studio (Gemini): x-goog-api-key header; the SDKs that pass + # ``?key=`` as a query param are covered by match_query, which + # scans every query parameter for the token value. + "GEMINI_API_KEY": { + "hosts": ("generativelanguage.googleapis.com",), + "match_headers": ("x-goog-api-key",), + "aliases": ("GOOGLE_API_KEY",), + }, } -# Providers whose env-var names we recognize but whose API uses a non-bearer -# auth scheme (x-api-key, AAD/OAuth, SigV4, custom signatures). When any of -# these env vars are present at proxy-start time AND -# ``proxy.fail_on_uncovered_providers`` is true (default), ``start_proxy`` -# refuses to start. Without this list the sandbox would still hold real -# credentials for these providers and silently bypass the proxy. +# Providers whose env-var names we recognize but whose auth genuinely cannot +# be swapped by a static header/query replacement (SigV4 request signing, +# OAuth tokens minted by an SDK from a service-account file). Presence is +# surfaced as a warning at setup/status time — these are generic cloud creds +# that are usually present for unrelated tooling (terraform, gcloud, aws-cli), +# so they never block the proxy from starting. # -# Bare strings here are env-var names; the proxy doesn't try to wire them up, -# only flags their presence so the operator knows isolation is incomplete. +# NOTE: this list used to include Anthropic / Azure OpenAI / Gemini, with an +# LLM-specific fail-closed tier (``proxy.fail_on_uncovered_providers``). +# Those providers moved to ``_HEADER_AUTH_PROVIDERS`` once we wired +# ``match_headers`` (upstream confirmed support on the pinned v0.39.0), which +# emptied the fail-closed tier — the flag and its refuse-start path were +# deleted rather than kept as a dead toggle. _NON_BEARER_PROVIDERS: Tuple[str, ...] = ( - # Azure OpenAI: api-key header + optional AAD bearer. - "AZURE_OPENAI_API_KEY", # AWS Bedrock / SageMaker: SigV4-signed requests. "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", - # GCP Vertex AI: OAuth bearer from gcloud SDK, not a static env key. + # GCP Vertex AI: OAuth bearer minted by the SDK from a service-account + # file, not a static env key. "GOOGLE_APPLICATION_CREDENTIALS", ) @@ -181,6 +236,18 @@ "172.16.0.0/12", # RFC1918 "192.168.0.0/16", # RFC1918 "fc00::/7", # IPv6 ULA + # IPv4-mapped IPv6 (``::ffff:0:0/96``) covers the dual-stack case + # where an upstream resolves to e.g. ``::ffff:169.254.169.254`` and + # the kernel hands the v4-mapped form to the socket — that would + # otherwise be a clean SSRF bypass to IMDS through the v6 path. + "::ffff:0:0/96", + # RFC6598 / CGNAT — used by AWS VPC for shared services, K8s pod + # networks, many cloud overlays. Not strictly RFC1918 but operators + # universally want it denied for the same reasons. + "100.64.0.0/10", + # RFC2544 benchmark range — rare in practice but occasionally used + # for internal services and never legitimate as an upstream. + "198.18.0.0/15", ) @@ -265,11 +332,23 @@ class TokenMapping: When Bitwarden is configured as the credential source for the proxy, iron-proxy's *own* environment is populated from bws on startup — the sandbox still sees only ``proxy_token``. + + ``match_headers`` names the request headers iron-proxy scans for the + proxy token (default: ``Authorization`` for bearer providers; e.g. + ``("x-api-key", "Authorization")`` for Anthropic native). + + ``alias_env_names`` are additional env-var names the SANDBOX receives + the same proxy token under (e.g. ``GOOGLE_API_KEY`` for + ``GEMINI_API_KEY``). They do not appear in the iron-proxy config — + only one secrets rule is emitted per mapping, keyed on + ``real_env_name``. """ proxy_token: str real_env_name: str upstream_hosts: Tuple[str, ...] + match_headers: Tuple[str, ...] = ("Authorization",) + alias_env_names: Tuple[str, ...] = () # --------------------------------------------------------------------------- @@ -411,6 +490,15 @@ def install_iron_proxy(*, force: bool = False) -> Path: _http_download(asset_url, archive_path) _http_download(checksum_url, checksum_path) + # Defense-in-depth (maxpetrusenko P1): verify the GPG signature of + # checksums.txt before trusting it. The archive download honors ambient + # proxy env (urllib), so a compromised channel could serve a matching + # binary + checksums pair; the detached signature + pinned public key + # close that release-channel tamper gap. Best-effort: if gpg or the + # signature assets aren't available we log and fall back to the SHA-256 + # check alone rather than hard-failing offline installs. + _verify_checksums_signature(tmp, checksum_path) + expected = _expected_sha256(checksum_path, asset_name) actual = _sha256_file(archive_path) if expected.lower() != actual.lower(): @@ -448,6 +536,12 @@ def install_iron_proxy(*, force: bool = False) -> Path: ) os.replace(staged, target) + # Invalidate the version cache so a freshly-installed binary + # re-probes ``--version`` on the next ``get_status()`` call instead + # of returning the pre-upgrade string. Long-lived processes that + # bump the pinned version via ``force=True`` need this. + _VERSION_CACHE.pop(str(target), None) + logger.info("Installed iron-proxy %s at %s", _IRON_PROXY_VERSION, target) return target @@ -462,6 +556,79 @@ def _http_download(url: str, dest: Path) -> None: raise RuntimeError(f"Failed to download {url}: {exc}") from exc +def _verify_checksums_signature(tmp: Path, checksum_path: Path) -> bool: + """Best-effort GPG verification of ``checksums.txt`` (maxpetrusenko P1). + + Downloads the detached signature (``checksums.txt.asc``) and the release + signing key (``public-key.asc``), imports the key into an ephemeral + keyring, and verifies the signature over ``checksum_path``. + + Returns True when the signature is verified. Returns False (with a warning) + when verification is unavailable — ``gpg`` not installed, or the signature / + public-key assets are missing from the release. Raises RuntimeError ONLY + when verification actively FAILS (a present-but-bad signature), which is a + tamper signal we must not ignore. + + Rationale for graceful degradation on "unavailable": the SHA-256 check + against ``checksums.txt`` remains in force regardless, and many install + hosts (CI, minimal containers) won't have gpg. We harden when we can and + never make gpg a hard dependency for a working install. + """ + gpg = shutil.which("gpg") + if not gpg: + logger.warning( + "gpg not found on PATH — skipping iron-proxy release-signature " + "verification (SHA-256 checksum check still enforced)." + ) + return False + + sig_url = f"{_IRON_PROXY_RELEASE_BASE}/{_IRON_PROXY_CHECKSUM_SIG_NAME}" + pubkey_url = f"{_IRON_PROXY_RELEASE_BASE}/{_IRON_PROXY_PUBKEY_NAME}" + sig_path = tmp / _IRON_PROXY_CHECKSUM_SIG_NAME + pubkey_path = tmp / _IRON_PROXY_PUBKEY_NAME + + try: + _http_download(sig_url, sig_path) + _http_download(pubkey_url, pubkey_path) + except RuntimeError as exc: + logger.warning( + "iron-proxy release signature assets unavailable (%s) — skipping " + "GPG verification (SHA-256 checksum check still enforced).", exc, + ) + return False + + # Ephemeral keyring so we never touch the user's real GPG home. + gnupg_home = tmp / "gnupg" + gnupg_home.mkdir(mode=0o700, exist_ok=True) + base_cmd = [gpg, "--homedir", str(gnupg_home), "--batch", "--no-tty"] + + imp = subprocess.run( # noqa: S603 — gpg path from trusted PATH lookup + [*base_cmd, "--import", str(pubkey_path)], + capture_output=True, timeout=60, + ) + if imp.returncode != 0: + logger.warning( + "Could not import iron-proxy signing key — skipping GPG " + "verification (SHA-256 still enforced): %s", + imp.stderr.decode("utf-8", "replace")[:200], + ) + return False + + verify = subprocess.run( # noqa: S603 + [*base_cmd, "--verify", str(sig_path), str(checksum_path)], + capture_output=True, timeout=60, + ) + if verify.returncode != 0: + # A present signature that does NOT verify is a tamper signal — fail hard. + raise RuntimeError( + "iron-proxy checksums.txt failed GPG signature verification — " + "refusing to install (possible release-channel tampering). " + f"gpg: {verify.stderr.decode('utf-8', 'replace')[:300]}" + ) + logger.info("Verified iron-proxy checksums.txt GPG signature.") + return True + + def _expected_sha256(checksum_file: Path, asset_name: str) -> str: """Parse the standard ``sha256sum`` output: `` ``.""" @@ -522,16 +689,36 @@ def iron_proxy_version(binary: Path) -> str: return cached try: - res = subprocess.run( # noqa: S603 — binary path is trusted + # Build a minimal env: only PATH, HOME, and locale vars. + # The version probe is a one-shot subprocess — forwarding + # the full host env (OPENAI_API_KEY, ANTHROPIC_API_KEY, etc.) + # to a PATH-resolved or unverified binary is an unnecessary + # credential leak. Reuse the same allowlist the daemon + # subprocess uses (see _build_proxy_subprocess_env). + minimal_env: Dict[str, str] = {} + parent = os.environ + for name in _PROXY_SUBPROCESS_ENV_ALLOWLIST: + if name in parent: + minimal_env[name] = parent[name] + # The S603 warning is legitimate for the PATH-fallback case + # (find_iron_proxy → shutil.which), but --version with a + # scrubbed env is safe regardless of binary provenance. + res = subprocess.run( # noqa: S603 [str(binary), "--version"], capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=_RUN_TIMEOUT, + env=minimal_env, ) except (OSError, subprocess.TimeoutExpired): return "" out = (res.stdout or res.stderr or "").strip() - _VERSION_CACHE[key] = out + # Don't cache empty output — that would poison ``hermes egress + # status`` for the lifetime of the process if the first probe hit a + # corrupt binary or a flag-rename in a newer upstream. Re-probe on + # the next call instead. + if out: + _VERSION_CACHE[key] = out return out @@ -651,42 +838,221 @@ def mint_proxy_token(prefix: str = "hermes-proxy") -> str: return f"{prefix}-{hashlib.sha256(os.urandom(32)).hexdigest()[:32]}" +def _management_token_path() -> Path: + return _proxy_state_dir() / "management.token" + + +def ensure_management_token(*, force: bool = False) -> str: + """Return the management-API bearer key, minting it on first call. + + Stored at ``/proxy/management.token`` with 0600 perms. + The daemon receives it via the ``HERMES_IRON_PROXY_MGMT_KEY`` env var + (named in the generated config's ``management.api_key_env``); + ``hermes egress reload`` reads the same file to authenticate. + """ + + p = _management_token_path() + if not force and p.exists(): + try: + existing = p.read_text(encoding="utf-8").strip() + if existing: + return existing + except OSError: + pass + token = mint_proxy_token(prefix="hermes-mgmt") + fd = os.open( + str(p), + os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + os.fchmod(fd, 0o600) + except (OSError, AttributeError): + pass + try: + os.write(fd, token.encode("utf-8")) + finally: + os.close(fd) + return token + + +def _read_management_token() -> Optional[str]: + p = _proxy_state_dir_ro() / "management.token" + try: + token = p.read_text(encoding="utf-8").strip() + except OSError: + return None + return token or None + + +def _read_management_listen_from_config( + config_path: Optional[Path] = None, +) -> Optional[Tuple[str, int]]: + """Return ``(host, port)`` of the management listener, if configured.""" + + cfg = config_path or (_proxy_state_dir_ro() / "proxy.yaml") + if not cfg.exists(): + return None + try: + import yaml + except ImportError: + return None + try: + data = yaml.safe_load(cfg.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return None + listen = ((data or {}).get("management") or {}).get("listen") or "" + if not isinstance(listen, str) or ":" not in listen: + return None + host, _, port_s = listen.rpartition(":") + try: + port = int(port_s) + except ValueError: + return None + return (host or "127.0.0.1", port) + + +def reload_proxy() -> bool: + """Hot-reload the running daemon's ruleset via the management API. + + POSTs to ``/v1/reload`` on the loopback management listener; the daemon + re-reads proxy.yaml and atomically swaps the transform pipeline — + validation failures leave the running config untouched (HTTP 422). + + Returns True on a successful reload. Raises ``RuntimeError`` with an + actionable message when the daemon isn't running, the config predates + management-API support (no ``management`` block → restart required), + or the reload is rejected. + """ + + pid = _read_pid() + if not pid or not _pid_alive(pid): + raise RuntimeError( + "iron-proxy is not running — nothing to reload. " + "Run `hermes egress start`." + ) + mgmt = _read_management_listen_from_config() + if mgmt is None: + raise RuntimeError( + "The generated proxy.yaml has no management listener (written " + "before reload support). Re-run `hermes egress setup` and use " + "`hermes egress restart` this one time." + ) + token = _read_management_token() + if not token: + raise RuntimeError( + "management.token is missing — re-run `hermes egress setup`, " + "then `hermes egress restart`." + ) + + import urllib.error + import urllib.request + + host, port = mgmt + req = urllib.request.Request( + f"http://{host}:{port}/v1/reload", + method="POST", + headers={"Authorization": f"Bearer {token}"}, + data=b"", + ) + try: + with urllib.request.urlopen(req, timeout=_MGMT_RELOAD_TIMEOUT) as resp: + if resp.status == 200: + return True + raise RuntimeError( + f"management API returned unexpected status {resp.status}" + ) + except urllib.error.HTTPError as exc: + body = "" + try: + body = exc.read().decode("utf-8", errors="replace")[:500] + except OSError: + pass + if exc.code == 422: + raise RuntimeError( + f"iron-proxy rejected the new config (validation failed; " + f"the running ruleset is unchanged): {body}" + ) from exc + if exc.code == 401: + raise RuntimeError( + "management API rejected our key (401). The running " + "daemon was started with a different management.token — " + "run `hermes egress restart`." + ) from exc + raise RuntimeError( + f"management reload failed (HTTP {exc.code}): {body}" + ) from exc + except (urllib.error.URLError, OSError) as exc: + # A daemon started from a pre-management config is alive but has + # no listener on the management port. + raise RuntimeError( + f"could not reach the management API at {host}:{port} ({exc}). " + "If the daemon was started before reload support, run " + "`hermes egress restart` once." + ) from exc + + def _default_http_listen(tunnel_port: int) -> List[str]: - """Build the list of host:port pairs the proxy should bind on. - - Always binds loopback (``127.0.0.1``) so host-side test tooling can hit - the proxy directly. On Linux we also bind the docker bridge gateway - (``172.17.0.1`` by default) so containers can reach the proxy via - ``host.docker.internal:host-gateway``. We do NOT bind ``0.0.0.0`` — - that would expose the proxy (and, with a leaked sandbox token, the - user's API quota) to anyone on the local network. - - On macOS / Windows Docker Desktop the bridge gateway is managed by - Desktop itself and ``host.docker.internal`` resolves via VPNkit, so - a single loopback bind is enough. + """Build the single host:port bind the proxy should listen on. + + iron-proxy v0.39 supports exactly ONE ``proxy.http_listen`` bind per + daemon process, so this returns a one-element list and the choice of + host matters: + + * **Linux:** bind the docker bridge gateway (``172.17.0.1`` by + default). Sandboxes reach the proxy via + ``host.docker.internal:host-gateway``, which Docker resolves to + exactly this bridge gateway IP on Linux — a loopback-only bind is + unreachable from inside containers there. The bridge IP is still + host-local (it's an address on the host's ``docker0`` interface), + so host-side tooling and the status probe can reach it too. When + no docker bridge is detected (docker not installed / not started), + fall back to loopback — there are no sandboxes to serve in that + state, and the operator gets a warning. + * **macOS / Windows Docker Desktop:** ``host.docker.internal`` + resolves via VPNkit to the host, so a loopback bind is reachable + from containers and is the least-exposed choice. + + We never bind ``0.0.0.0`` — that would expose the proxy (and, with a + leaked sandbox token, the user's API quota) to anyone on the local + network. The bridge-gateway bind is reachable by other containers + on the default bridge network, which is unavoidable given v0.39's + single-bind limit; requests still require a minted proxy token and + an allowlisted upstream. """ - binds = [f"127.0.0.1:{tunnel_port}"] if platform.system() == "Linux": bridge_ip = _detect_docker_bridge_ip() if bridge_ip and bridge_ip != "127.0.0.1": - binds.append(f"{bridge_ip}:{tunnel_port}") - return binds + return [f"{bridge_ip}:{tunnel_port}"] + logger.warning( + "No docker bridge (docker0) detected — binding iron-proxy to " + "loopback only. Docker sandboxes will NOT be able to reach " + "the proxy until it is restarted with docker running." + ) + return [f"127.0.0.1:{tunnel_port}"] def _detect_docker_bridge_ip() -> Optional[str]: """Return the docker0 bridge IPv4, if present, else None. - Best-effort: we try ``ip -4 addr show docker0`` first, then fall back - to parsing ``/proc/net/route`` for the bridge IP. Anything that fails - or doesn't look like an IPv4 returns None — callers handle that as - "no bridge bind". + Best-effort: we try ``ip -4 addr show docker0`` first. Anything that + fails, doesn't parse as a strict IPv4, or parses as an address we + must NOT bind to (unspecified, loopback, multicast, reserved, public) + returns None — callers handle that as "no bridge bind". + + SECURITY: a hostile ``ip`` shim earlier on the operator's PATH used + to be able to inject ``0.0.0.0`` here and re-open INADDR_ANY binding + that the rest of the bind-policy work explicitly closed. We + validate via :mod:`ipaddress` and reject anything that isn't + plausibly a docker bridge IP (private + non-special). """ + candidate: Optional[str] = None try: res = subprocess.run( # noqa: S603 — ip is a system binary ["ip", "-4", "-o", "addr", "show", "docker0"], - capture_output=True, text=True, timeout=2, + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2, ) if res.returncode == 0: for line in res.stdout.splitlines(): @@ -694,13 +1060,44 @@ def _detect_docker_bridge_ip() -> Optional[str]: # Expected: ": docker0 inet 172.17.0.1/16 ..." for i, tok in enumerate(parts): if tok == "inet" and i + 1 < len(parts): - ip = parts[i + 1].split("/")[0] - # cheap sanity: four dotted parts. - if ip.count(".") == 3: - return ip + candidate = parts[i + 1].split("/")[0] + break + if candidate is not None: + break except (OSError, subprocess.TimeoutExpired): - pass - return None + return None + + if not candidate: + return None + + # Stdlib validation: rejects garbage strings AND special-purpose + # addresses that must not be used as a bind target. + try: + addr = ipaddress.IPv4Address(candidate) + except (ipaddress.AddressValueError, ValueError): + return None + # Reject: + # - 0.0.0.0 / INADDR_ANY (is_unspecified) + # - 127.0.0.0/8 (is_loopback — already in deny list) + # - 224.0.0.0/4 (is_multicast) + # - 240.0.0.0/4 (is_reserved) + # - 169.254.0.0/16 (is_link_local — IMDS range, never docker0) + # - global / public IPs (is_global — docker0 must be RFC1918) + if ( + addr.is_unspecified + or addr.is_loopback + or addr.is_multicast + or addr.is_reserved + or addr.is_link_local + or addr.is_global + ): + logger.warning( + "Refusing suspicious docker bridge IP %s reported by `ip`; " + "skipping bridge bind.", candidate, + ) + return None + + return str(addr) def build_proxy_config( @@ -720,13 +1117,14 @@ def build_proxy_config( real secrets from its OWN environment via ``source: {type: env, var: ...}``; the sandbox never sees them. - Bind policy: by default we bind loopback (``127.0.0.1``) plus the - docker bridge gateway IP on Linux (``172.17.0.1`` or whatever - ``docker0`` resolves to). Sandboxes use ``host.docker.internal`` which - Linux Docker maps to the bridge gateway via ``--add-host``; macOS / - Windows Docker Desktop manage their own gateway. We do NOT bind - ``0.0.0.0`` — a LAN peer with a leaked sandbox token could otherwise - spend the operator's API quota against any allowlisted upstream. + Bind policy: the sandbox-facing listeners (``tunnel_listen`` on + ``tunnel_port``, plain-HTTP ``http_listen`` on ``tunnel_port + 1``) + bind the docker bridge gateway on Linux (``172.17.0.1`` or whatever + ``docker0`` resolves to — that's what ``host.docker.internal`` + resolves to inside containers there) and loopback on macOS / Windows + Docker Desktop. We do NOT bind ``0.0.0.0`` — a LAN peer with a + leaked sandbox token could otherwise spend the operator's API quota + against any allowlisted upstream. SSRF policy: ``upstream_deny_cidrs`` defaults to a conservative deny list covering loopback, link-local (incl. AWS/GCP/Azure IMDS at @@ -740,8 +1138,8 @@ def build_proxy_config( CONNECT tunnel. We point it at loopback so it doesn't conflict with anything else and disable the listener. * The ``proxy.tunnel_listen`` is what sandboxes hit via ``HTTPS_PROXY``. - iron-proxy's ``http_listen`` is for DNS-routed MITM HTTP/HTTPS flows; - CONNECT/SOCKS clients need the dedicated tunnel listener. + ``http_listen`` / ``https_listen`` are present (loopback only) so the + proxy boots; sandboxes never route directly to them. * ``allowlist`` transform takes ``domains:`` and ``cidrs:``, not ``hosts:``. * ``secrets`` transform takes ``secrets:`` (plural), each with a ``source``, a ``replace.proxy_value`` (the sandbox-visible token), and @@ -756,17 +1154,35 @@ def build_proxy_config( secrets_rules = [] for m in mappings: - match_headers = list(_PROVIDER_MATCH_HEADERS.get(m.real_env_name, ("Authorization",))) + match_headers = list(m.match_headers or ("Authorization",)) secrets_rules.append({ "source": {"type": "env", "var": m.real_env_name}, "replace": { "proxy_value": m.proxy_token, + # Per-provider header set: bearer providers match only + # Authorization; header-auth providers (Anthropic native + # x-api-key, Azure api-key, Gemini x-goog-api-key) match + # their native header (+ Authorization where the provider + # also accepts bearer flows). v0.39 matches header names + # case-insensitively — see parseHeaderMatchers upstream. "match_headers": match_headers, - # The token is also accepted as a bearer query param in case - # the sandbox passes it that way. Body matching is off — we + # The token is also accepted as a query param — v0.39 scans + # every query parameter for the token value, which covers + # SDKs that pass ``?key=`` (Gemini) as well as + # bearer-in-query styles. Body matching is off — we # don't want body inspection forced for every request. "match_query": True, "match_body": False, + # Fail closed (maxpetrusenko P1): when a request reaches an + # allowlisted upstream WITHOUT the proxy token present in a + # matched location, reject it instead of forwarding as-is. + # Without this, a real provider key that a sandbox process + # sent directly (not via the minted token) would still pass + # the proxy boundary to the allowed host. With require=true, + # iron-proxy returns ActionReject when no token swap fired + # (v0.39 secrets transform: replaceConfig.Require, enforced in + # TransformRequest — verified present in the pinned version). + "require": True, }, "rules": [{"host": h} for h in m.upstream_hosts], }) @@ -780,20 +1196,50 @@ def build_proxy_config( else: deny_cidrs = list(upstream_deny_cidrs) - # Listen address. iron-proxy v0.39 accepts one CONNECT/SOCKS5 - # tunnel_listen scalar; it does not accept the later multi-listen shape. - # Use the first detected safe bind target so generated config actually - # boots against the pinned binary. ``http_listen`` stays loopback/ephemeral - # because Hermes sandboxes use HTTPS_PROXY, not DNS interception. + # Listen addresses. iron-proxy v0.39 takes a single string per + # listener field — there is no plural ``http_listens`` form, despite + # earlier drafts of this module claiming v0.39 accepts both. An + # empirical strings(1) audit + a live "start the binary and observe + # the YAML unmarshal error" confirms the singular form is the only + # one the binary accepts. + # + # LISTENER ROLES (verified live against the v0.39 binary): + # * ``tunnel_listen`` is the CONNECT + MITM listener. HTTPS through + # ``HTTPS_PROXY`` issues CONNECT — this is the listener sandboxes + # must reach. A CONNECT sent to ``http_listen`` is NOT terminated: + # v0.39 forwards it upstream as a regular request and the upstream + # responds 400. + # * ``http_listen`` is the absolute-form plain-HTTP forward listener + # (``HTTP_PROXY`` for ``http://`` URLs). Transforms fire here too. + # Both get the sandbox-facing bind host: tunnel on ``tunnel_port``, + # plain HTTP on ``tunnel_port + 1``. + # + # The bind host comes from _default_http_listen: the docker bridge + # gateway on Linux (containers reach the proxy via + # host.docker.internal, which maps to the bridge gateway there — + # loopback would be unreachable from inside sandboxes) and loopback + # on macOS/Windows Docker Desktop (where host.docker.internal routes + # to the host via VPNkit). listens = list(http_listen) if http_listen else _default_http_listen(tunnel_port) primary_listen = listens[0] if listens else f"127.0.0.1:{tunnel_port}" + bind_host = primary_listen.rsplit(":", 1)[0] or "127.0.0.1" + plain_http_listen = f"{bind_host}:{tunnel_port + 1}" - # iron-proxy v0.39 only accepts log.level. Keep audit_log as a reserved - # parameter because the CLI pre-creates the file and a future pinned binary - # may support audit_path, but do not emit unsupported YAML that prevents the - # current managed binary from booting. - _ = audit_log log_block: Dict = {"level": "info"} + # NOTE: ``log.audit_path`` is NOT a field in iron-proxy v0.39's + # ``config.Log`` struct — the binary rejects it with + # ``field audit_path not found in type config.Log``. Per-request + # audit records are written to the same log destination as + # everything else at this binary version; the operator-facing + # ``audit.log`` file we pre-create is still useful as a sentinel + # for monitoring (logrotate target, downstream tail watchers) but + # the daemon does not write to it directly. The kwarg is kept so + # we're forward-compatible with a future v0.40+ that adds the + # field; if you upgrade _IRON_PROXY_VERSION and the upstream gains + # ``log.audit_path``, re-enable the line below. + # if audit_log is not None: + # log_block["audit_path"] = str(audit_log) + _ = audit_log # consumed by ensure_audit_log() / docs only on v0.39 return { # DNS section is required by the binary's config parser, but we run @@ -805,13 +1251,20 @@ def build_proxy_config( "proxy_ip": "127.0.0.1", }, "proxy": { - # Hermes sandboxes use standard HTTPS_PROXY / CONNECT semantics, - # which iron-proxy serves from tunnel_listen. Keep the DNS-routed - # http/https MITM listeners on ephemeral loopback ports so they do - # not collide with the operator-facing tunnel port. - "http_listen": "127.0.0.1:0", - "https_listen": "127.0.0.1:0", + # tunnel_listen is the CONNECT/MITM listener — what sandboxes + # hit via `HTTPS_PROXY=http://host:tunnel_port` for HTTPS + # upstreams (curl/requests/node issue CONNECT through it). + # http_listen handles absolute-form plain-HTTP forwards + # (`HTTP_PROXY` for http:// URLs) on tunnel_port+1. Both + # bind the docker bridge gateway on Linux / loopback on + # Docker Desktop — NEVER 0.0.0.0. LAN peers with a leaked + # sandbox token would otherwise be able to spend the + # operator's API quota against any allowlisted upstream. "tunnel_listen": primary_listen, + "http_listen": plain_http_listen, + # The HTTPS-listener (direct TLS termination, no CONNECT) + # gets a loopback ephemeral port — we don't expose it. + "https_listen": "127.0.0.1:0", "max_request_body_bytes": 16 * 1024 * 1024, "max_response_body_bytes": 0, "upstream_response_header_timeout": "120s", @@ -819,6 +1272,32 @@ def build_proxy_config( # default. An empty list opts out entirely. "upstream_deny_cidrs": deny_cidrs, }, + # iron-proxy v0.39 starts a Prometheus-style metrics server by + # default on ``:9090`` — which is the SAME port as our default + # ``tunnel_port: 9090``, causing a guaranteed bind collision on + # startup. Pin the metrics listener to an ephemeral loopback + # port (``127.0.0.1:0``) so the metrics binding can't collide + # with the proxy listener regardless of what tunnel_port the + # operator chose. NOTE: ``:0`` means the kernel picks a fresh + # random port each start and nothing records it — metrics are + # effectively disabled/undiscoverable at this pin. If we want + # scrapable metrics later, allocate a fixed port and surface it + # in ``ProxyStatus`` / ``hermes egress status``. + "metrics": { + "listen": "127.0.0.1:0", + }, + # Operator-facing management API — loopback only, bearer-key + # authenticated (key read from the env var named below; injected + # by ``start_proxy`` from ``management.token``). ``POST /v1/reload`` + # re-reads THIS config file and atomically swaps the transform + # pipeline — `hermes egress reload` applies allowlist/token/mapping + # changes without a restart. Loopback deliberately: sandboxes must + # never reach the management surface, so it does NOT bind the + # docker bridge like the traffic listeners do. + "management": { + "listen": f"127.0.0.1:{tunnel_port + _MGMT_PORT_OFFSET}", + "api_key_env": _MGMT_API_KEY_ENV, + }, "tls": { "ca_cert": str(ca_cert), "ca_key": str(ca_key), @@ -836,18 +1315,25 @@ def build_proxy_config( }, ], "log": log_block, - "metrics": {"listen": "127.0.0.1:0"}, } def ensure_audit_log(audit_path: Path) -> None: """Create the audit log file with private permissions (0o600). - Called from the wizard right before ``start_proxy``. Without this, - iron-proxy creates the file under the default umask the first time it - writes — meaning every host-side request history is potentially - world-readable. We pre-create the file empty with 0o600 so the daemon - inherits the tight permissions. + Called from the wizard right before ``start_proxy``. On the pinned + v0.39 the daemon never writes this file (no ``log.audit_path`` + config field), so the pre-create is purely forward-compat: when the + pin moves to a version that supports a dedicated audit stream, the + file already exists with tight permissions and the daemon inherits + them instead of creating it under the default umask. + + Raises :class:`RuntimeError` on any OSError (planted symlink, + immutable parent dir, full disk) so the caller can decide how to + surface it. The wizard treats this as a WARNING on v0.39 — the + file is non-load-bearing until the version bump — but the qualified + message keeps operators from wiring monitoring to a path that can't + exist. """ try: @@ -863,7 +1349,11 @@ def ensure_audit_log(audit_path: Path) -> None: finally: os.close(fd) except OSError as exc: - logger.warning("Could not pre-create audit log %s: %s", audit_path, exc) + raise RuntimeError( + f"Refusing to start: could not pre-create audit log " + f"{audit_path} with restrictive permissions ({exc}). " + f"Move or chmod any existing file at that path and retry." + ) from exc def write_proxy_config(config: Dict) -> Path: @@ -885,8 +1375,13 @@ def write_proxy_config(config: Dict) -> Path: tmp_path = state / ".proxy.yaml.tmp" with open(tmp_path, "w", encoding="utf-8") as f: yaml.safe_dump(config, f, default_flow_style=False, sort_keys=False) + # Tighten perms on the temp file BEFORE the atomic replace so the + # final path is never briefly world-readable under a slack umask + # (the config embeds proxy token values). chmod-after-replace would + # leave a TOCTOU window; the 0o700 state dir mitigates but same-uid + # processes could still race. + os.chmod(tmp_path, stat.S_IRUSR | stat.S_IWUSR) os.replace(tmp_path, out) - os.chmod(out, stat.S_IRUSR | stat.S_IWUSR) return out @@ -907,6 +1402,8 @@ def write_mappings(mappings: List[TokenMapping]) -> Path: "proxy_token": m.proxy_token, "env_name": m.real_env_name, "upstream_hosts": list(m.upstream_hosts), + "match_headers": list(m.match_headers), + "alias_env_names": list(m.alias_env_names), } for m in mappings ], @@ -914,8 +1411,11 @@ def write_mappings(mappings: List[TokenMapping]) -> Path: tmp_path = state / ".mappings.json.tmp" with open(tmp_path, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) + # chmod before the atomic replace — see write_proxy_config. The + # mappings file holds proxy token values, so close the TOCTOU window + # rather than chmod-ing after the file is already at its final path. + os.chmod(tmp_path, stat.S_IRUSR | stat.S_IWUSR) os.replace(tmp_path, out) - os.chmod(out, stat.S_IRUSR | stat.S_IWUSR) return out @@ -938,6 +1438,11 @@ def load_mappings() -> List[TokenMapping]: proxy_token=item["proxy_token"], real_env_name=item["env_name"], upstream_hosts=tuple(item.get("upstream_hosts") or ()), + # Pre-header-auth mappings.json files (written before the + # match_headers/alias fields existed) load with the bearer + # defaults — identical to their behavior at write time. + match_headers=tuple(item.get("match_headers") or ("Authorization",)), + alias_env_names=tuple(item.get("alias_env_names") or ()), )) except (KeyError, TypeError): continue @@ -970,6 +1475,24 @@ def discover_provider_mappings( real_env_name=env_name, upstream_hosts=hosts, )) + for env_name, spec in _HEADER_AUTH_PROVIDERS.items(): + aliases = tuple(spec.get("aliases") or ()) + # A mapping is minted when the canonical name OR any alias is + # available. Aliases collapse into ONE mapping (single secrets + # rule) because two require-rules on the same host would reject + # each other's requests. The canonical env name is what + # iron-proxy reads — when only the alias is set in the host env, + # the subprocess-env builder mirrors it (see + # ``_build_proxy_subprocess_env``). + if env_name not in names and not any(a in names for a in aliases): + continue + mappings.append(TokenMapping( + proxy_token=mint_proxy_token(prefix=env_name.lower().replace("_api_key", "")), + real_env_name=env_name, + upstream_hosts=tuple(spec["hosts"]), + match_headers=tuple(spec["match_headers"]), + alias_env_names=aliases, + )) return mappings @@ -979,14 +1502,14 @@ def discover_uncovered_providers( ) -> List[str]: """Return env-var names for providers we recognize but can't proxy. - Anthropic native (x-api-key), AWS Bedrock (SigV4), Azure OpenAI - (api-key), etc. When any of these are configured, the sandbox is - holding real credentials that the proxy can't strip — the isolation - guarantee is incomplete for those providers. + AWS Bedrock (SigV4) and GCP Vertex (SDK-minted OAuth) can't be swapped + by a static header replacement. When any of these are configured, the + sandbox is holding real credentials that the proxy can't strip — the + isolation guarantee is incomplete for those providers. - The wizard uses this to print a warning at setup time; ``start_proxy`` - can be configured to refuse to start when ``fail_on_uncovered_providers`` - is true. + The wizard and ``hermes egress status`` use this to print a warning. + (Anthropic / Azure OpenAI / Gemini used to be here; they're now + first-class swapped providers via ``_HEADER_AUTH_PROVIDERS``.) """ if available_env_names is not None: @@ -1024,12 +1547,15 @@ def merge_mappings( for d in discovered: prior = by_name.get(d.real_env_name) if prior is not None and not rotate: - # Preserve the token, refresh the host list in case we added - # new upstreams since last setup. + # Preserve the token; refresh hosts/headers/aliases in case + # the provider spec changed since last setup (new upstreams, + # a provider moving from uncovered to header-auth, etc). out.append(TokenMapping( proxy_token=prior.proxy_token, real_env_name=prior.real_env_name, upstream_hosts=d.upstream_hosts, + match_headers=d.match_headers, + alias_env_names=d.alias_env_names, )) else: out.append(d) @@ -1092,6 +1618,50 @@ def _pid_proc_starttime(pid: int) -> Optional[str]: return fields[19] +def _persisted_nonce_path() -> Path: + """Path to the on-disk sibling of the pidfile that stores the nonce. + + Written by ``_write_pidfile_safely`` after ``start_proxy`` plants + the nonce in the iron-proxy child env, read by ``_pid_alive`` in a + later CLI invocation (``stop`` / ``status``) so cross-process + PID-recycling defense holds. + """ + return _proxy_state_dir_ro() / "iron-proxy.nonce" + + +def _read_persisted_nonce() -> Optional[str]: + """Read the on-disk nonce written next to the pidfile. + + Returns None when the file is missing, unreadable, or empty — + callers fall back to argv0 basename matching in that case. + """ + p = _persisted_nonce_path() + try: + # O_NOFOLLOW: defence-in-depth against a planted symlink at the + # nonce path; same-uid required to plant one but worth defending + # since the nonce read here decides whether stop_proxy will + # SIGKILL a candidate PID. + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(str(p), flags) + except OSError: + return None + try: + # Ownership check — if the file isn't owned by us, ignore it. + # Same threat model as the pidfile uid check. + try: + st = os.fstat(fd) + if hasattr(os, "getuid") and st.st_uid != os.getuid(): + return None + except AttributeError: + pass + data = os.read(fd, 256).decode("utf-8", errors="ignore").strip() + return data or None + finally: + os.close(fd) + + def _pid_alive(pid: int) -> bool: """Return True iff ``pid`` is alive AND is an iron-proxy process. @@ -1130,12 +1700,27 @@ def _pid_alive(pid: int) -> bool: # Strong proof: nonce env var matches. /proc//environ is null- # separated KEY=VALUE pairs; substring search is safe. + # + # The nonce can come from either: + # 1. the module-global ``_proxy_nonce`` set during this process's + # own ``start_proxy`` call (same-process case); + # 2. the on-disk ``iron-proxy.nonce`` file written by + # ``_write_pidfile_safely``, used when ``start`` and ``stop`` + # run in separate CLI invocations (cross-process case). + # Either source provides the same defeat-PID-recycling guarantee. + nonce_candidates: List[str] = [] if _proxy_nonce: + nonce_candidates.append(_proxy_nonce) + on_disk = _read_persisted_nonce() + if on_disk and on_disk not in nonce_candidates: + nonce_candidates.append(on_disk) + if nonce_candidates: try: env_bytes = Path(f"/proc/{pid}/environ").read_bytes() - needle = f"{_HERMES_IRON_PROXY_NONCE_ENV}={_proxy_nonce}".encode() - if needle in env_bytes: - return True + for nonce in nonce_candidates: + needle = f"{_HERMES_IRON_PROXY_NONCE_ENV}={nonce}".encode() + if needle in env_bytes: + return True except OSError: pass @@ -1158,7 +1743,7 @@ def _pid_alive(pid: int) -> bool: try: res = subprocess.run( # noqa: S603 ["ps", "-p", str(pid), "-o", "comm="], - capture_output=True, text=True, timeout=2, + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2, ) if res.returncode == 0: comm = (res.stdout or "").strip() @@ -1176,6 +1761,7 @@ def start_proxy( binary: Optional[Path] = None, config_path: Optional[Path] = None, extra_env: Optional[Dict[str, str]] = None, + install_if_missing: bool = True, refresh_secrets_from_bitwarden: bool = False, bitwarden_config: Optional[Dict] = None, ) -> ProxyStatus: @@ -1198,7 +1784,7 @@ def start_proxy( if existing and _pid_alive(existing): return get_status() - bin_path = binary or find_iron_proxy(install_if_missing=True) + bin_path = binary or find_iron_proxy(install_if_missing=install_if_missing) if bin_path is None: raise RuntimeError( "iron-proxy binary not available — run `hermes egress install`." @@ -1222,6 +1808,13 @@ def start_proxy( bitwarden_config=bitwarden_config, ) + # If the generated config enables the management API, the daemon + # validates at startup that the api_key_env is non-empty. Inject the + # persisted key (minting it if this is a config written by a newer + # setup but the token file was removed). + if _read_management_listen_from_config(cfg) is not None: + env[_MGMT_API_KEY_ENV] = ensure_management_token() + # Plant a per-start nonce in the child env so ``_pid_alive`` can # confirm a candidate PID still refers to *our* binary across PID # recycling. Module-global is fine — only one managed proxy per @@ -1235,15 +1828,37 @@ def start_proxy( # immediately after Popen (the child has its own dup). Without the # close-on-success path, every restart leaked one fd in the Hermes # process. - log_fd = os.open( - str(log_path), - os.O_WRONLY | os.O_CREAT | os.O_APPEND, - 0o600, - ) + # + # O_NOFOLLOW (defence-in-depth, same threat model as the pidfile + # path): a same-uid attacker who plants ``iron-proxy.log`` as a + # symlink to e.g. ``~/.ssh/authorized_keys`` would otherwise cause + # every restart to append daemon diagnostics to that file. + log_open_flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND + if hasattr(os, "O_NOFOLLOW"): + log_open_flags |= os.O_NOFOLLOW + try: + log_fd = os.open(str(log_path), log_open_flags, 0o600) + except OSError as exc: + # ELOOP from a planted symlink — refuse with a clear error. + raise RuntimeError( + f"Refusing to write iron-proxy log {log_path}: {exc}. " + "Remove that path manually and retry." + ) from exc try: os.fchmod(log_fd, 0o600) # tighten if file pre-existed except OSError: pass + # Verify ownership — same st_uid check the pidfile uses. + try: + st = os.fstat(log_fd) + if hasattr(os, "getuid") and st.st_uid != os.getuid(): + os.close(log_fd) + raise RuntimeError( + f"iron-proxy log {log_path} has unexpected owner " + f"uid={st.st_uid}; refusing to write." + ) + except AttributeError: + pass # Windows try: # Use the fd directly via the dup mechanism; Popen will dup() it @@ -1274,50 +1889,163 @@ def start_proxy( except OSError: pass + # Write the pidfile IMMEDIATELY after Popen, BEFORE the listening + # verification. If the parent dies during the poll loop (SIGINT, + # OOM, kernel pause), the pidfile is still on disk so the next + # ``hermes egress stop`` can clean up the orphan. Failure paths + # below unlink the pidfile when they kill the child. + pidfile = _pidfile() + try: + _write_pidfile_safely(pidfile, proc.pid) + except RuntimeError: + # Kill the orphan so we don't leave a daemon nobody can stop. + _kill_and_wait(proc, grace_seconds=2) + raise + # Poll-with-timeout instead of an unconditional 5s sleep. The Go # binary normally comes up in <200ms; falling through within 100ms # of liveness keeps Docker container creation snappy. - tunnel_port = _read_tunnel_port_from_config() or _DEFAULT_TUNNEL_PORT - deadline = time.time() + _STARTUP_GRACE_SECONDS - while time.time() < deadline: - if proc.poll() is not None: - tail = _tail_log(log_path, lines=20) - raise RuntimeError( - f"iron-proxy exited immediately (code {proc.returncode}). " - f"Last log lines:\n{tail}" - ) - if _port_listening("127.0.0.1", tunnel_port): - break - time.sleep(0.1) + # + # We scope a Ctrl-C handler around the poll loop so an operator who + # hits Ctrl-C while waiting for ``hermes egress start`` doesn't leak + # an orphan with the port bound. + # + # Probe the CONFIGURED bind host, not loopback unconditionally — on + # Linux the daemon binds the docker bridge gateway, where a loopback + # connect never succeeds and we'd kill a healthy daemon as "never + # came up". + listen_hp = _read_http_listen_from_config() + if listen_hp is not None: + probe_host, tunnel_port = listen_hp + else: + probe_host, tunnel_port = "127.0.0.1", _DEFAULT_TUNNEL_PORT + listening = False + def _interrupt_handler(_signum, _frame): # pragma: no cover - signal path + # Kill the child and unlink the pidfile, then re-raise so the + # caller sees the interrupt. + _kill_and_wait(proc, grace_seconds=2) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise KeyboardInterrupt() + + prev_sigint = None + prev_sigterm = None + install_handlers = ( + platform.system() != "Windows" + and threading.current_thread() is threading.main_thread() + ) + if install_handlers: + prev_sigint = signal.signal(signal.SIGINT, _interrupt_handler) + prev_sigterm = signal.signal(signal.SIGTERM, _interrupt_handler) + try: + deadline = time.time() + _STARTUP_GRACE_SECONDS + # Do-while shape: check listening at least once even when the + # grace window is 0 (test harness / synchronous fast-path). + while True: + if proc.poll() is not None: + tail = _tail_log(log_path, lines=20) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise RuntimeError( + f"iron-proxy exited immediately (code {proc.returncode}). " + f"Last log lines:\n{tail}" + ) + if _port_listening(probe_host, tunnel_port): + listening = True + break + if time.time() >= deadline: + break + time.sleep(0.1) + finally: + if install_handlers: + signal.signal(signal.SIGINT, prev_sigint) + signal.signal(signal.SIGTERM, prev_sigterm) + + # Final exit check — process may have died right at deadline. if proc.poll() is not None: tail = _tail_log(log_path, lines=20) + try: + pidfile.unlink() + except FileNotFoundError: + pass raise RuntimeError( f"iron-proxy exited immediately (code {proc.returncode}). " f"Last log lines:\n{tail}" ) - pidfile = _pidfile() - # Use os.open with O_NOFOLLOW to refuse to follow a pre-existing - # symlink at the pidfile path (defence-in-depth; same-uid required to - # plant a symlink but worth defending). O_TRUNC clobbers any stale - # content. - open_flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + # The previous version of this code treated "process still alive at + # deadline" as success. That left iron-proxy running but + # non-listening on the port, with a pidfile pointing at it — + # subsequent restarts would fail with "address in use" because the + # orphan still held the port. Require port-listening for success. + if not listening: + tail = _tail_log(log_path, lines=20) + _kill_and_wait(proc, grace_seconds=2) + try: + pidfile.unlink() + except FileNotFoundError: + pass + raise RuntimeError( + f"iron-proxy did not bind {probe_host}:{tunnel_port} within " + f"{_STARTUP_GRACE_SECONDS}s. Process was killed. " + f"Last log lines:\n{tail}" + ) + + logger.info("Started iron-proxy pid=%s config=%s", proc.pid, cfg) + return get_status() + + +def _write_pidfile_safely(pidfile: Path, pid: int) -> None: + """Write ``pid`` to ``pidfile`` with O_EXCL + O_NOFOLLOW + ownership check. + + O_EXCL means "another start is in progress" if the file already + exists with a live owner — we cleanly fail rather than racing. When + the existing pidfile points at a dead pid (stale crash), we + explicitly unlink it before retrying once. + + Side effect: also persists the in-process nonce to disk so + cross-CLI-invocation ``_pid_alive`` checks (start in one process, + stop in another) can still defeat PID recycling. + """ + open_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): open_flags |= os.O_NOFOLLOW try: fd = os.open(str(pidfile), open_flags, 0o600) + except FileExistsError: + # Pidfile already exists. If it points at a live iron-proxy, + # caller's _read_pid + _pid_alive at the top of start_proxy + # should already have returned. Reaching here means EITHER + # the previous _pid_alive check raced (rare; another start in + # flight), OR a stale pidfile survived a crash. Discriminate + # and retry once with O_TRUNC if stale. + existing_pid = _read_pid() + if existing_pid and _pid_alive(existing_pid): + raise RuntimeError( + f"Another iron-proxy start appears to be in progress " + f"(pidfile {pidfile} -> pid {existing_pid}). " + f"Run `hermes egress stop` if that proxy is stuck." + ) + # Stale — unlink and retry. + try: + pidfile.unlink() + except FileNotFoundError: + pass + fd = os.open(str(pidfile), open_flags, 0o600) except OSError as exc: - # If the file existed as a symlink, O_NOFOLLOW returns ELOOP. - # Surface a clear error and let the operator clean up. + # ELOOP from a planted symlink at the pidfile path. raise RuntimeError( f"Refusing to write pidfile {pidfile}: {exc}. " "Remove that path manually and retry." ) from exc + try: - # Verify the file we just opened is owned by us. On POSIX, - # st_uid mismatch means a same-uid race won and we got a hostile - # file — bail rather than write the pid into it. + # Ownership check — same st_uid pattern the log file uses. try: st = os.fstat(fd) if hasattr(os, "getuid") and st.st_uid != os.getuid(): @@ -1326,55 +2054,52 @@ def start_proxy( ) except AttributeError: pass # Windows - os.write(fd, str(proc.pid).encode("utf-8")) + os.write(fd, str(pid).encode("utf-8")) finally: os.close(fd) - logger.info("Started iron-proxy pid=%s config=%s", proc.pid, cfg) - return get_status() - - -def _load_hermes_env_values(names: set[str]) -> Dict[str, str]: - """Return selected values from /.env without mutating os.environ. + # Persist the nonce next to the pidfile (sibling, 0o600). + # ``stop_proxy`` in a separate CLI invocation can read this and use + # it to confirm the pid still refers to our binary even though the + # module-global ``_proxy_nonce`` is fresh in the new process. + if _proxy_nonce: + noncefile = pidfile.with_suffix(".nonce") + nfd = -1 + try: + nopen = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + nopen |= os.O_NOFOLLOW + nfd = os.open(str(noncefile), nopen, 0o600) + os.write(nfd, _proxy_nonce.encode("utf-8")) + except OSError: + # Best-effort. Without the nonce file we fall back to + # argv0-basename matching, which is what we did before. + pass + finally: + if nfd >= 0: + try: + os.close(nfd) + except OSError: + pass - The egress proxy intentionally forwards only mapped provider secrets into - the child process. CLI commands may discover those mappings from Hermes' - .env file even when the current shell has not exported the variables, so - start-up needs the same narrow .env fallback to make the swap work. - """ - if not names: - return {} +def _kill_and_wait(proc: "subprocess.Popen", *, grace_seconds: int = 2) -> None: + """Best-effort SIGTERM → wait → SIGKILL for a child we own.""" try: - from hermes_constants import get_hermes_home - env_path = get_hermes_home() / ".env" - except Exception: # noqa: BLE001 - best-effort fallback only - return {} - if not env_path.exists(): - return {} - values: Dict[str, str] = {} - try: - lines = env_path.read_text(encoding="utf-8").splitlines() - except UnicodeDecodeError: - lines = env_path.read_text(encoding="latin-1").splitlines() + proc.terminate() except OSError: - return {} - for raw in lines: - line = raw.strip() - if not line or line.startswith("#") or "=" not in line: - continue - if line.startswith("export "): - line = line[len("export "):].lstrip() - key, value = line.split("=", 1) - key = key.strip() - if key not in names: - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: - value = value[1:-1] - if value: - values[key] = value - return values + return + try: + proc.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + try: + proc.kill() + except OSError: + pass + try: + proc.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + pass def _build_proxy_subprocess_env( @@ -1405,16 +2130,24 @@ def _build_proxy_subprocess_env( # The proxy reads the real upstream secrets from its OWN env, indexed # by ``m.real_env_name`` in the YAML config's ``secrets.source.var`` - # field. Forward those — but only those. Hermes secrets commonly live - # in /.env rather than the operator's shell, so fall back to - # that file when the parent process did not export a mapped key. - needed = {m.real_env_name for m in load_mappings()} - env_file_values = _load_hermes_env_values(needed) + # field. Forward those — but only those. For alias providers + # (GEMINI_API_KEY / GOOGLE_API_KEY), the rule is keyed on the canonical + # name; when only the alias is set in the host env, mirror its value + # into the canonical name so the swap still has a real secret. + alias_sources: Dict[str, Tuple[str, ...]] = {} + needed = set() + for m in load_mappings(): + needed.add(m.real_env_name) + if m.alias_env_names: + alias_sources[m.real_env_name] = tuple(m.alias_env_names) for name in needed: if name in parent: env[name] = parent[name] - elif name in env_file_values: - env[name] = env_file_values[name] + else: + for alias in alias_sources.get(name, ()): + if parent.get(alias): + env[name] = parent[alias] + break # Optional Bitwarden refresh path. Pulled lazily so the proxy module # doesn't hard-depend on the bitwarden module being importable in @@ -1437,9 +2170,35 @@ def _build_proxy_subprocess_env( # Only inject env names we have a mapping for — extra # secrets in the BW project shouldn't leak into the proxy # process unless they're going to be used by the swap. + missing = sorted(needed - set(secrets)) for n in needed: if n in secrets: env[n] = secrets[n] + if missing: + # stephenschoettler #1: don't silently keep stale + # host-env values when BWS mode was explicitly + # selected. An operator on credential_source=bitwarden + # picked it specifically to get rotation; falling back + # to parent env reintroduces the bug class the mode + # is supposed to defeat. ``allow_env_fallback`` is the + # documented, deliberate opt-out — honor it here exactly + # as the empty-token branch below does (the error + # message tells operators to set it, so it must work). + if not (bitwarden_config or {}).get("allow_env_fallback"): + raise RuntimeError( + f"Bitwarden refresh did not return secrets for " + f"{missing}. Either add the secrets to your BWS " + f"project, switch to credential_source: env via " + f"`hermes egress setup --no-bitwarden`, or set " + f"`proxy.allow_env_fallback: true` in config.yaml " + f"to opt into the legacy host-env fallback." + ) + logger.warning( + "Bitwarden refresh did not return secrets for %s — " + "falling back to host env for those names " + "(allow_env_fallback=true).", + missing, + ) # bws warnings are non-secret status messages (e.g. "no # project found", "rate limited"), but the taint analyzer # can't tell that — log the count and let the operator @@ -1451,15 +2210,45 @@ def _build_proxy_subprocess_env( len(warnings), ) else: + # NOTE: deliberately do not interpolate access_token_name + # in the log message — CodeQL's taint analyzer treats + # bitwarden_config values as secret-tainted (it can't + # distinguish the env-var NAME from the env-var VALUE). + # The name is non-secret but logging it just trips the + # check for no real benefit. + if not (bitwarden_config or {}).get("allow_env_fallback"): + raise RuntimeError( + "credential_source=bitwarden but the access-token " + "env or project_id is empty. Either set both, " + "switch to credential_source: env, or set " + "`proxy.allow_env_fallback: true` to opt into " + "the legacy fallback behaviour." + ) logger.warning( - "credential_source=bitwarden but access_token_env=%s or " - "project_id is empty — proxy will fall back to parent env", - access_token_name, + "credential_source=bitwarden but access-token env or " + "project_id is empty — proxy will fall back to parent env " + "(allow_env_fallback=true).", ) - except (ImportError, RuntimeError) as exc: + except (ImportError,) as exc: + # The BWS module or one of its runtime deps isn't importable. + # Mirror the sibling branches: if allow_env_fallback isn't + # explicitly enabled, fail closed — credential_source=bitwarden + # with a unavailable module should not silently degrade to host + # env. A wizard-time check can't catch a dependency that goes + # missing between setup and a later restart. + if not (bitwarden_config or {}).get("allow_env_fallback"): + raise RuntimeError( + "Bitwarden refresh module unavailable at proxy start " + "(credential_source=bitwarden with " + "proxy.allow_env_fallback: false). Either fix the " + "import, switch to credential_source: env, or set " + "`proxy.allow_env_fallback: true` to opt into the " + "legacy fallback behaviour." + ) from exc logger.warning( - "Bitwarden refresh failed at proxy start, falling back to " - "parent env: %s", exc, + "Bitwarden refresh module unavailable at proxy start, " + "falling back to parent env (allow_env_fallback=true): %s", + exc, ) # Caller-supplied overrides win. This is intentionally last so the @@ -1481,9 +2270,19 @@ def stop_proxy() -> bool: global _proxy_nonce + def _cleanup_state_files() -> None: + """Best-effort cleanup of pidfile + persisted nonce.""" + _pidfile().unlink(missing_ok=True) + try: + _persisted_nonce_path().unlink() + except FileNotFoundError: + pass + except OSError: + pass + pid = _read_pid() if not pid or not _pid_alive(pid): - _pidfile().unlink(missing_ok=True) + _cleanup_state_files() _proxy_nonce = None return False @@ -1495,7 +2294,7 @@ def stop_proxy() -> bool: try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: - _pidfile().unlink(missing_ok=True) + _cleanup_state_files() _proxy_nonce = None return False @@ -1527,7 +2326,7 @@ def stop_proxy() -> bool: except ProcessLookupError: pass - _pidfile().unlink(missing_ok=True) + _cleanup_state_files() _proxy_nonce = None logger.info("Stopped iron-proxy pid=%s", pid) return True @@ -1543,7 +2342,12 @@ def get_status() -> ProxyStatus: """ status = ProxyStatus() - status.tunnel_port = _read_tunnel_port_from_config() or _DEFAULT_TUNNEL_PORT + listen_hp = _read_http_listen_from_config() + if listen_hp is not None: + probe_host, status.tunnel_port = listen_hp + else: + probe_host = "127.0.0.1" + status.tunnel_port = _DEFAULT_TUNNEL_PORT binary = find_iron_proxy(install_if_missing=False) if binary: @@ -1564,12 +2368,33 @@ def get_status() -> ProxyStatus: pid = _read_pid() if pid and _pid_alive(pid): status.pid = pid - status.listening = _port_listening("127.0.0.1", status.tunnel_port) + # Probe the configured bind host — on Linux that's the docker + # bridge gateway, where a loopback connect would report a healthy + # daemon as "not listening". + status.listening = _port_listening(probe_host, status.tunnel_port) return status def _read_tunnel_port_from_config() -> Optional[int]: + listen = _read_http_listen_from_config() + if listen is None: + return None + return listen[1] + + +def _read_http_listen_from_config() -> Optional[Tuple[str, int]]: + """Return ``(host, port)`` of the configured sandbox-facing listener. + + Reads ``proxy.tunnel_listen`` — the CONNECT/MITM listener sandboxes + hit via ``HTTPS_PROXY`` — falling back to ``proxy.http_listen`` for + configs written before the tunnel/http listener-role split. + + The bind host matters for liveness probes: on Linux the daemon binds + the docker bridge gateway (e.g. ``172.17.0.1``), where a loopback + connect would report "not listening" for a perfectly healthy daemon. + """ + cfg = _proxy_state_dir_ro() / "proxy.yaml" if not cfg.exists(): return None @@ -1581,16 +2406,20 @@ def _read_tunnel_port_from_config() -> Optional[int]: data = yaml.safe_load(cfg.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError): return None + proxy_block = (data or {}).get("proxy") or {} # The CLI/Docker side calls this "the tunnel port" because that's how - # sandboxes use it (HTTPS_PROXY). On the iron-proxy side, standard - # HTTP CONNECT/SOCKS5 proxy traffic belongs on proxy.tunnel_listen. - listen = ((data or {}).get("proxy") or {}).get("tunnel_listen") or "" + # sandboxes use it (HTTPS_PROXY) — on the iron-proxy side it's the + # tunnel_listen (CONNECT + MITM). http_listen is the plain-HTTP + # forward listener on tunnel_port+1. + listen = proxy_block.get("tunnel_listen") or proxy_block.get("http_listen") or "" if not isinstance(listen, str) or ":" not in listen: return None + host, _, port_s = listen.rpartition(":") try: - return int(listen.rsplit(":", 1)[1]) + port = int(port_s) except ValueError: return None + return (host or "127.0.0.1", port) def _port_listening(host: str, port: int) -> bool: @@ -1621,9 +2450,23 @@ def _tail_log(path: Path, *, lines: int = 20) -> str: def _reset_for_tests() -> None: - """No-op today — kept symmetric with bitwarden._reset_cache_for_tests.""" + """Clear module-level caches so tests get a fresh start. + + This module owns two mutable globals that need reset between tests: + - ``_VERSION_CACHE`` — subprocess output cache keyed by binary path. + - ``_proxy_nonce`` — the strong-proof token written by ``start_proxy`` + and read by ``_pid_alive`` to defeat PID recycling. + + Today the repo's tests run each file in its own subprocess (per + AGENTS.md) so leakage is bounded, but any in-process caller + (notebooks, ad-hoc scripts, ``pytest -p no:xdist``) would otherwise + see whichever values were probed first regardless of subsequent + ``install_iron_proxy(force=True)`` or ``start_proxy`` calls. + """ - return None + global _proxy_nonce + _VERSION_CACHE.clear() + _proxy_nonce = None # Make a small set of symbols available without underscored access. @@ -1635,6 +2478,7 @@ def _reset_for_tests() -> None: "discover_uncovered_providers", "ensure_audit_log", "ensure_ca_cert", + "ensure_management_token", "find_iron_proxy", "get_status", "install_iron_proxy", @@ -1642,6 +2486,7 @@ def _reset_for_tests() -> None: "load_mappings", "merge_mappings", "mint_proxy_token", + "reload_proxy", "start_proxy", "stop_proxy", "write_mappings", diff --git a/agent/reasoning_timeouts.py b/agent/reasoning_timeouts.py index 13df836b1100..da7fcd2fcbed 100644 --- a/agent/reasoning_timeouts.py +++ b/agent/reasoning_timeouts.py @@ -102,9 +102,18 @@ # ``claude-opus-4`` so non-thinking Claude 3.x or future # non-reasoning Claude variants don't match. ("claude-opus-4", 240), + ("claude-opus-5", 240), ("claude-sonnet-5", 180), ("claude-sonnet-4.5", 180), ("claude-sonnet-4.6", 180), + # Anthropic Mythos-class named reasoning models (claude-fable-5, …). + # 1M context + 128K output — heavier thinking phase than the + # numbered Claude line, so the floor is in the deep-reasoning tier + # alongside o1 / deepseek-r1 / nemotron-3-ultra. Without this + # entry the stale-stream detector kills fable-5's thinking phase + # at the default 180s (300s with context scaling), tripping the + # cross-turn circuit breaker after 5 consecutive stale kills. + ("claude-fable", 600), # xAI Grok reasoning variants. Explicit reasoning-only keys # plus one for the ``non-reasoning`` variant so users picking # the fast variant don't get the 300s floor. Bare ``grok-3``, @@ -137,19 +146,18 @@ # so we accept that community forks inheriting the same prefix are # treated as reasoning models (a reasonable default — the upstream # gateway timing is the same). -_PATTERN_CACHE: dict[str, re.Pattern[str]] = {} - - -def _get_pattern(slug: str) -> re.Pattern[str]: - compiled = _PATTERN_CACHE.get(slug) - if compiled is None: - compiled = re.compile( - r"^" - + re.escape(slug) - + r"(?:$|[\-._])" - ) - _PATTERN_CACHE[slug] = compiled - return compiled +# Pre-compile all patterns at module load time to avoid per-call regex +# compilation and thread-safety issues with the mutable _PATTERN_CACHE. +# The list is built once at import and never mutated afterwards, so it is +# safe for free-threaded Python 3.13+ without any locking. The slug is kept +# in each entry for debuggability (log/inspection), even though _match_any +# only consumes floor + pattern. +_SORTED_REASONING_FLOORS: list[tuple[str, float, re.Pattern[str]]] = [ + (slug, floor, re.compile(r"^" + re.escape(slug) + r"(?:$|[\-._])")) + for slug, floor in sorted( + _REASONING_STALE_TIMEOUT_FLOORS, key=lambda kv: -len(kv[0]) + ) +] def _match_any(model_lower: str) -> Optional[float]: @@ -160,13 +168,8 @@ def _match_any(model_lower: str) -> Optional[float]: order is irrelevant: longest slug wins (so ``o3-mini`` beats ``o3`` on a model like ``openai/o3-mini``). """ - # Sort by slug length descending so longer / more-specific slugs - # win on shared prefixes (o3-mini beats o3). - sorted_floors = sorted( - _REASONING_STALE_TIMEOUT_FLOORS, key=lambda kv: -len(kv[0]) - ) - for slug, floor in sorted_floors: - if _get_pattern(slug).search(model_lower): + for _slug, floor, pattern in _SORTED_REASONING_FLOORS: + if pattern.search(model_lower): return float(floor) return None @@ -206,6 +209,8 @@ def get_reasoning_stale_timeout_floor(model: object) -> Optional[float]: 300.0 >>> get_reasoning_stale_timeout_floor("anthropic/claude-opus-4-6") 240.0 + >>> get_reasoning_stale_timeout_floor("anthropic/claude-fable-5") + 600.0 >>> get_reasoning_stale_timeout_floor("gpt-4o") is None True >>> get_reasoning_stale_timeout_floor("olmo-1") is None diff --git a/agent/redact.py b/agent/redact.py index ebca1ae75f14..ea70246a9079 100644 --- a/agent/redact.py +++ b/agent/redact.py @@ -111,6 +111,23 @@ r"fw-[A-Za-z0-9]{30,}", # Fireworks AI API key r"fw_[A-Za-z0-9]{30,}", # Fireworks AI API key r"fpk_[A-Za-z0-9]{30,}", # Fireworks AI project key + # GitLab token families (each pattern keeps a full literal prefix so the + # _PREFIX_SUBSTRINGS pre-screen stays false-negative-free). Ported from + # openclaw/openclaw#112954; follow-up invited in #4541. + r"glpat-[A-Za-z0-9_\-]{10,}", # GitLab personal access token + r"gloas-[A-Za-z0-9_\-]{10,}", # GitLab OAuth application secret + r"gldt-[A-Za-z0-9_\-]{10,}", # GitLab deploy token + r"glrt-[A-Za-z0-9_.\-]{10,}", # GitLab runner authentication token (routable tokens are dotted) + r"glrtr-[A-Za-z0-9_.\-]{10,}", # GitLab runner registration token (routable) + r"glcbt-[A-Za-z0-9_\-]{10,}", # GitLab CI/CD job token + r"glptt-[A-Za-z0-9_\-]{10,}", # GitLab pipeline trigger token + r"glft-[A-Za-z0-9_\-]{10,}", # GitLab feed token + r"glimt-[A-Za-z0-9_\-]{10,}", # GitLab incoming mail token + r"glagent-[A-Za-z0-9_\-]{10,}", # GitLab agent (KAS) token + r"glsoat-[A-Za-z0-9_\-]{10,}", # GitLab service-account access token + r"glffct-[A-Za-z0-9_\-]{10,}", # GitLab feature-flags client token + r"glwt-[A-Za-z0-9_\-]{10,}", # GitLab workspace token + r"GR1348941[A-Za-z0-9_\-]{10,}", # GitLab legacy runner registration token ] # ENV assignment patterns: KEY=value where KEY contains a secret-like name. @@ -140,6 +157,11 @@ # The colon-form URL guard (skip when ``://`` present) lives at the call site. _SECRET_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential|auth)" _CFG_VALUE = r"(['\"]?)([^\s&]+?)\2(?=[\s&]|$)" +# Linear pre-gate for the _CFG_*_RE subs below: a text with no secret keyword +# can never match either pattern, so the (potentially backtrack-heavy) subs +# are skipped entirely for such text. See the call site in +# redact_sensitive_text(). +_CFG_SECRET_WORD_RE = re.compile(_SECRET_CFG_NAMES, re.IGNORECASE) # Programmatic env lookups (``os.getenv(...)``, ``os.environ[...]``, # ``os.environ.get(...)``, ``process.env.X``, ``$ENV{X}``) reference variable @@ -149,9 +171,13 @@ r"^(?:os\.(?:getenv|environ)|process\.env|\$ENV\{)" ) # Namespaced (dotted) key: the secret word may sit anywhere in a dotted path. +# NOTE(perf): possessive quantifiers (py3.11+) replace the nested quantifier +# ``(?:[A-Za-z0-9_\-]+\.)+`` (exponential backtracking on long dotted runs). +# The ``*`` runs bordering {_SECRET_CFG_NAMES} must stay backtrackable +# (secret words are matchable by the class, e.g. ``app.api.key=…``). _CFG_DOTTED_RE = re.compile( - rf"((?:[A-Za-z0-9_\-]+\.)+[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*" - rf"|[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*\.[A-Za-z0-9_.\-]+)" + rf"([A-Za-z0-9_\-]++\.[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*+" + rf"|[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*\.[A-Za-z0-9_.\-]++)" rf"={_CFG_VALUE}", re.IGNORECASE, ) @@ -170,11 +196,92 @@ # is masked by _AUTH_HEADER_RE); ``auth_token``/``auth-token`` still match via # the ``token`` keyword. Quoted values defer to _JSON_FIELD_RE via the lookahead. _YAML_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential)" +# NOTE(perf): possessive quantifiers wherever the successor is disjoint; the +# leading ``[A-Za-z0-9_.\-]*`` stays backtrackable (see _CFG_DOTTED_RE note). _YAML_ASSIGN_RE = re.compile( - rf"(^[ \t]*[A-Za-z0-9_.\-]*{_YAML_CFG_NAMES}[A-Za-z0-9_.\-]*)(:[ \t]*)(?!['\"])([^\s&]+)", + rf"(^[ \t]*+[A-Za-z0-9_.\-]*{_YAML_CFG_NAMES}[A-Za-z0-9_.\-]*+)(:[ \t]*+)(?!['\"])([^\s&]++)", re.IGNORECASE | re.MULTILINE, ) +# Word-boundary validation for the mixed/lowercase key patterns above +# (_CFG_DOTTED_RE, _CFG_ANCHORED_RE, _YAML_ASSIGN_RE). +# +# Those key classes allow arbitrary alphanumeric affixes around the secret +# keyword so real key names like ``client_secret``, ``clientSecret``, and +# ``s3.secret-key`` match. The side effect: ordinary prose/document words that +# merely CONTAIN a keyword also matched — ``Secretary: J.Smith`` (secret), +# ``tokenizer: cl100k_base`` (token), ``author=Smith`` (auth) — mangling +# legitimate content on the surfaces that run these passes (browser snapshots, +# log lines, kanban summaries, CLI-echoed command output). Ported from +# nearai/ironclaw#6129, where the same substring false positive ("Secretary of +# the Treasury" matching the ``secret`` marker) scrubbed legitimate tool +# results from the replayed transcript and sent the model into a re-fetch +# loop. +# +# A keyword occurrence only counts when it sits at a word boundary within the +# key: at the key's edge, next to a non-letter (``_ - . 3``), or at a +# camelCase transition (``clientSecret``, ``secretKey``, ``APIToken``). A +# trailing plural ``s`` is treated as part of the keyword (``secrets:``, +# ``tokens:``). Common concatenated compounds keep matching via explicit +# alternatives (``authtoken`` ngrok, ``authkey`` tailscale, ``secretkey`` +# minio, ``apikey``). Embedded occurrences inside a larger word +# (``secretary``, ``tokenizer``, ``authored``, ``credentialing``) no longer +# match. ALL-CAPS keys keep the legacy embedded matching (``MYTOKEN=…``) — an +# all-caps key is almost never prose, the same rationale as _ENV_ASSIGN_RE. +_KEY_KEYWORD_RE = re.compile( + r"(?:api|auth|access|refresh|session|secret)[ _.\-]?(?:key|token)" + r"|token|secret|passwd|password|credential|auth", + re.IGNORECASE, +) + + +def _is_word_start(s: str, i: int) -> bool: + """True if position ``i`` in ``s`` begins a word (not mid-word).""" + if i == 0: + return True + prev, cur = s[i - 1], s[i] + if not prev.isalpha(): + return True + if cur.isupper() and prev.islower(): + return True # camelCase: clientSecret + # Acronym run ending: APIToken — the 'T' begins a new word when it is + # followed by lowercase while the preceding run is uppercase. + if cur.isupper() and prev.isupper() and i + 1 < len(s) and s[i + 1].islower(): + return True + return False + + +def _is_word_end(s: str, j: int, *, allow_plural: bool = True) -> bool: + """True if position ``j`` (exclusive end) in ``s`` ends a word.""" + if j >= len(s): + return True + cur = s[j] + if not cur.isalpha(): + return True + if cur.isupper() and s[j - 1].islower(): + return True # camelCase continuation: secretKey + if allow_plural and cur in "sS": + return _is_word_end(s, j + 1, allow_plural=False) + return False + + +def _key_has_secret_keyword(key: str) -> bool: + """True if ``key`` contains a secret keyword at a word boundary. + + Post-match validator for _CFG_DOTTED_RE / _CFG_ANCHORED_RE / + _YAML_ASSIGN_RE hits — rejects prose words that merely embed a keyword + (``secretary``, ``tokenizer``, ``authored``). Safe to call with the + _ENV_ASSIGN_RE key too: all-caps keys short-circuit to the legacy + embedded-match behavior. + """ + letters = [c for c in key if c.isalpha()] + if letters and all(c.isupper() for c in letters): + return True # legacy all-caps behavior (MYTOKEN=…) + for m in _KEY_KEYWORD_RE.finditer(key): + if _is_word_start(key, m.start()) and _is_word_end(key, m.end()): + return True + return False + # JSON field patterns: "apiKey": "value", "token": "value", etc. _JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)" _JSON_FIELD_RE = re.compile( @@ -298,8 +405,17 @@ # Match userinfo in both absolute (``scheme://user:pass@host``) and # network-path (``//user:pass@host``) references. The authority boundary stops # at path/query/fragment delimiters so an ``@`` elsewhere in a URL is ignored. +# +# Anchored on the mandatory ``//`` rather than an optional scheme prefix: the +# scheme sits outside the match either way (replacement callbacks re-emit +# group(1), so ``https:`` stays untouched in the surrounding text), and the +# old optional-scheme prefix ``(?:[A-Za-z][A-Za-z0-9+.-]*:)?`` backtracked +# catastrophically (O(n²)) on long unbroken alphanumeric runs — a 320KB +# synthetic compaction payload spent ~55s inside this pattern per sub() call. +# Output-equivalence to the old pattern was fuzz-verified (20k random strings +# plus targeted URL forms). _STRICT_URL_USERINFO_RE = re.compile( - r"((?:[A-Za-z][A-Za-z0-9+.-]*:)?//)([^/\s?#@]+)@" + r"(//)([^/\s?#@]+)@" ) # HTTP access logs often use a relative request target rather than a full URL: @@ -614,13 +730,28 @@ def _redact_env(m): # prose/log contexts (issue #2852): ``KEY=os.getenv('X')``. if _ENV_LOOKUP_VALUE_RE.match(value): return m.group(0) + # Keyword must sit at a word boundary within the key — + # ``author=Smith`` / ``press.secretary=…`` are prose, not + # credentials (ported from nearai/ironclaw#6129). All-caps + # keys (the _ENV_ASSIGN_RE shape) short-circuit to legacy + # embedded matching inside the helper. + if not _key_has_secret_keyword(name): + return m.group(0) return f"{name}={quote}{_mask_token(value)}{quote}" text = _ENV_ASSIGN_RE.sub(_redact_env, text) # Lowercase/dotted config keys (issue #16413). Skip URLs entirely — # web-URL query params are intentionally passed through (see note # near the bottom of this function); _DB_CONNSTR_RE still guards # connection-string passwords. - if "://" not in text: + # + # Extra gate: every _CFG_*_RE match requires a secret keyword in + # the key, so a text without any secret keyword cannot match — + # skipping is exact. This matters because _CFG_DOTTED_RE + # backtracks quadratically on long unbroken [A-Za-z0-9_.\-] runs + # (e.g. base64/hex blobs in compaction payloads); the linear + # keyword scan prevents that pathological path on secret-free + # text. + if "://" not in text and _CFG_SECRET_WORD_RE.search(text): text = _CFG_DOTTED_RE.sub(_redact_env, text) text = _CFG_ANCHORED_RE.sub(_redact_env, text) @@ -647,6 +778,11 @@ def _redact_yaml(m): # not a leaked secret value. if _ENV_LOOKUP_VALUE_RE.match(value): return m.group(0) + # Keyword must sit at a word boundary within the key — + # ``Secretary: J.Smith`` / ``tokenizer: cl100k_base`` are + # document text, not credentials (nearai/ironclaw#6129). + if not _key_has_secret_keyword(key): + return m.group(0) return f"{key}{sep}{_mask_token(value)}" text = _YAML_ASSIGN_RE.sub(_redact_yaml, text) diff --git a/agent/relay_llm.py b/agent/relay_llm.py new file mode 100644 index 000000000000..96a98d54d24b --- /dev/null +++ b/agent/relay_llm.py @@ -0,0 +1,1170 @@ +"""Core NeMo Relay adapters for physical Hermes provider attempts.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import logging +from collections.abc import Callable, Iterator +from types import SimpleNamespace +from typing import Any + +from agent import relay_runtime + +logger = logging.getLogger(__name__) + + +_PROVIDER_MESSAGE_EXTENSION_KEYS = frozenset( + {"reasoning_content", "reasoning_details"} +) +_RELAY_INTERNAL_PROVIDER_HEADERS = frozenset( + {"x-dynamo-parent-session-id", "x-dynamo-session-id"} +) + + +def execute( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run one non-streaming physical provider attempt through Relay.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + raw = callback_context.copy().run(callback, final_request) + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = _run_awaitable( + runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if _recover_successful_callback( + raw_response, + relay_error=exc, + callback_error=callback_error, + logical=logical, + defer_logical_completion=defer_logical_completion, + ): + return raw_response["value"] + raise + + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return _namespace(managed) + + +async def execute_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run one asynchronous physical provider attempt through Relay.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return await callback(request) + logical = _logical_parent(runtime, session, parent, metadata) + parent = logical[1] if logical is not None else parent + + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + raw_response: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + async def invoke(next_request: Any) -> Any: + nonlocal callback_error + try: + final_request = _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + async def call_provider() -> Any: + return await callback(final_request) + + task = callback_context.copy().run( + asyncio.create_task, + call_provider(), + ) + raw = await task + except BaseException as exc: + callback_error = exc + raise + raw_response["value"] = raw + raw_response["json"] = _jsonable(raw) + return raw_response["json"] + + try: + managed = await runtime.run_in_session_async( + session, + runtime.relay.llm.execute, + name, + relay_request, + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if _recover_successful_callback( + raw_response, + relay_error=exc, + callback_error=callback_error, + logical=logical, + defer_logical_completion=defer_logical_completion, + ): + return raw_response["value"] + raise + + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + if "value" in raw_response and _json_equal(managed, raw_response["json"]): + return raw_response["value"] + return _namespace(managed) + + +def execute_current( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run a provider attempt under the inherited Hermes turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return callback(request) + return execute( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +async def execute_current_async( + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> Any: + """Run an async provider attempt under the inherited turn when present.""" + turn = relay_runtime.active_turn() + if turn is None: + return await callback(request) + return await execute_async( + request, + callback, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +def _has_running_event_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def stream_current( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + name: str, + model_name: str, + finalizer: Callable[[], Any], + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, + completed_response_predicate: Callable[[Any], bool] | None = None, +) -> Any: + """Run a provider stream under the inherited Hermes turn when present. + + When ``completed_response_predicate`` is set and the stream_factory returns + a complete response instead of an iterator (e.g. AnthropicAuxiliaryClient + and other shims that ignore ``stream=True``), unwrap and return the + completed response directly. This mirrors the pre-Relay behavior where + ``call_llm(stream=True)`` returned the raw response and the consumer's + own ``hasattr(stream, "choices")`` check handled it (#11732, #55933) — + without the unwrap the response stays trapped as ``final_response`` on the + inner ManagedLlmStream and the outer consumer sees an empty stream. + """ + turn = relay_runtime.active_turn() + if turn is None: + return stream_factory(request) + if _has_running_event_loop(): + # Managed provider callbacks execute on the Relay session's event + # loop. A nested ManagedLlmStream built here would be synchronously + # iterated on that same loop thread, which asyncio forbids + # ("Cannot run the event loop while another loop is running"). + # Return the raw factory result instead: the outer managed stream + # already provides Relay tracking for the enclosing attempt, and its + # own completed_response_predicate traps a completed response (e.g. + # the MoA facade's auxiliary ``call_llm(stream=True)`` returning a + # full response when an adapter ignores ``stream=True``). + return stream_factory(request) + managed = stream( + request, + stream_factory, + session_id=turn.lease.session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + completed_response_predicate=completed_response_predicate, + ) + # In the non-managed path the factory already ran eagerly during __init__, + # so a completed response is visible immediately and must surface raw. + # In the managed path the factory runs lazily on first pull, so + # final_response is still None here and the managed stream is returned. + if completed_response_predicate is not None: + completed = getattr(managed, "final_response", None) + if completed is not None: + return completed + return managed + + +def stream( + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + finalizer: Callable[[], Any], + on_stream_created: Callable[[Any], None] | None = None, + on_chunk: Callable[[Any], None] | None = None, + chunk_adapter: Callable[[Any], Any] | None = None, + accept_chunk: Callable[[Any], bool] | None = None, + completed_response_predicate: Callable[[Any], bool] | None = None, + metadata: dict[str, Any] | None = None, + defer_logical_completion: bool = False, +) -> "ManagedLlmStream": + """Return a synchronous view of one Relay-managed provider stream.""" + return ManagedLlmStream( + request, + stream_factory, + session_id=session_id, + name=name, + model_name=model_name, + finalizer=finalizer, + on_stream_created=on_stream_created, + on_chunk=on_chunk, + chunk_adapter=chunk_adapter, + accept_chunk=accept_chunk, + completed_response_predicate=completed_response_predicate, + metadata=metadata, + defer_logical_completion=defer_logical_completion, + ) + + +class ManagedLlmStream(Iterator[Any]): + """Drive Relay's async stream from Hermes's provider worker thread.""" + + def __init__( + self, + request: dict[str, Any], + stream_factory: Callable[[dict[str, Any]], Any], + *, + session_id: str, + name: str, + model_name: str, + finalizer: Callable[[], Any], + on_stream_created: Callable[[Any], None] | None, + on_chunk: Callable[[Any], None] | None, + chunk_adapter: Callable[[Any], Any] | None, + accept_chunk: Callable[[Any], bool] | None, + completed_response_predicate: Callable[[Any], bool] | None, + metadata: dict[str, Any] | None, + defer_logical_completion: bool, + ) -> None: + self.final_response: Any = None + self._loop: asyncio.AbstractEventLoop | None = None + self._stream: Any = None + self._raw_stream_resource: Any = None + self._closed = False + self._close_error: BaseException | None = None + self._callback_error: BaseException | None = None + self._logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None = None + self._defer_logical_completion = defer_logical_completion + self._on_chunk = on_chunk + self._chunk_adapter = chunk_adapter or _namespace + self._accept_chunk = accept_chunk + self._relay_observes_chunks = False + self._provider_completed = False + self._raw_chunks: list[tuple[Any, Any]] = [] + self.output_modified = False + callback_context = contextvars.copy_context() + + def run_callback(callback: Callable[..., Any], *args: Any) -> Any: + # Relay can invoke stream surfaces while another callback still + # owns the captured Context. A fresh copy is safe to enter. + return callback_context.copy().run(callback, *args) + + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if ( + runtime is None + or session is None + or not runtime.managed_execution_enabled() + ): + raw_stream = stream_factory(request) + if completed_response_predicate is not None and completed_response_predicate( + raw_stream + ): + self.final_response = raw_stream + self._stream = iter(()) + else: + self._raw_stream_resource = raw_stream + if on_stream_created is not None: + on_stream_created(raw_stream) + self._stream = iter(raw_stream) + return + + self._logical = _logical_parent(runtime, session, parent, metadata) + if self._logical is not None: + parent = self._logical[1] + relay_request_body = _relay_request_body(request, metadata) + relay_request = runtime.relay.LLMRequest({}, relay_request_body) + codec_baseline_body = _codec_round_trip_request_body( + runtime.relay, + relay_request, + relay_request_body=relay_request_body, + metadata=metadata, + ) + + async def provider_stream(next_request: Any): + raw_stream = None + try: + raw_stream = run_callback( + stream_factory, + _provider_request( + request, + next_request, + relay_request_body=relay_request_body, + codec_baseline_body=codec_baseline_body, + metadata=metadata, + ) + ) + if ( + completed_response_predicate is not None + and run_callback( + completed_response_predicate, + raw_stream, + ) + ): + self.final_response = raw_stream + self._provider_completed = True + return + if on_stream_created is not None: + run_callback(on_stream_created, raw_stream) + raw_iterator = run_callback(iter, raw_stream) + while True: + try: + chunk = run_callback(next, raw_iterator) + except StopIteration: + break + if self._accept_chunk is not None and not run_callback( + self._accept_chunk, + chunk, + ): + break + encoded_chunk = _jsonable(chunk) + self._raw_chunks.append((encoded_chunk, chunk)) + yield encoded_chunk + self._provider_completed = True + except BaseException as exc: + self._callback_error = exc + raise + finally: + close = getattr(raw_stream, "close", None) + if callable(close): + try: + run_callback(close) + except BaseException as exc: + self._close_error = exc + raise + + def observe_chunk(chunk: Any) -> None: + if self._on_chunk is not None: + run_callback(self._on_chunk, _jsonable(chunk)) + + def relay_finalizer() -> Any: + # Relay can invoke the finalizer while unwinding a provider-stream + # failure. Preserve that original callback error instead of + # replacing it with a secondary "missing terminal response" error. + if self._callback_error is not None: + return None + try: + if self.final_response is not None: + return _jsonable(self.final_response) + return _jsonable(run_callback(finalizer)) + except BaseException as exc: + self._callback_error = exc + raise + + loop = asyncio.new_event_loop() + self._loop = loop + self._relay_observes_chunks = True + try: + self._stream = loop.run_until_complete( + runtime.run_in_session_async( + session, + runtime.relay.llm.stream_execute, + name, + relay_request, + provider_stream, + observe_chunk, + relay_finalizer, + handle=parent, + metadata=_jsonable(metadata or {}), + model_name=model_name, + codec=_codec(runtime.relay, metadata), + response_codec=_codec(runtime.relay, metadata), + ) + ) + except BaseException as exc: + if ( + isinstance(exc, Exception) + and self._provider_completed + and self._callback_error is None + ): + logger.warning( + "NeMo Relay stream post-processing failed after provider success; " + "preserving the provider result", + exc_info=True, + ) + self._preserve_pending_provider_chunks() + return + if not self._defer_logical_completion: + _complete_logical( + self._logical, + outcome="cancelled" if _is_cancellation(exc) else "failed", + ) + self._logical = None + loop.close() + self._loop = None + raise + + def __iter__(self) -> "ManagedLlmStream": + return self + + def __next__(self) -> Any: + if self._closed: + raise StopIteration + if self._loop is None: + try: + chunk = next(self._stream) + except StopIteration: + self._close(logical_outcome="cancelled") + raise + if self._accept_chunk is not None and not self._accept_chunk(chunk): + self._close(logical_outcome="cancelled") + raise StopIteration + return chunk + + async def next_chunk() -> Any: + return await anext(self._stream) + + try: + chunk = self._loop.run_until_complete(next_chunk()) + except StopAsyncIteration: + if self._raw_chunks: + self.output_modified = True + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome="success") + self._logical = None + self._close(logical_outcome="cancelled") + raise StopIteration from None + except BaseException as exc: + callback_error = self._callback_error + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + self._close(logical_outcome="failed") + raise callback_error + if ( + isinstance(exc, Exception) + and self._provider_completed + and callback_error is None + ): + logger.warning( + "NeMo Relay stream post-processing failed after provider success; " + "preserving the provider result", + exc_info=True, + ) + self._preserve_pending_provider_chunks() + return next(self) + self._close( + logical_outcome="cancelled" if _is_cancellation(exc) else "failed" + ) + raise + if not self._relay_observes_chunks and self._on_chunk is not None: + self._on_chunk(chunk) + for index, (encoded, raw) in enumerate(self._raw_chunks): + if _json_equal(chunk, encoded): + if index > 0: + self.output_modified = True + del self._raw_chunks[: index + 1] + return raw + self.output_modified = True + return self._chunk_adapter(chunk) + + def close(self) -> None: + """Close an explicitly abandoned stream and cancel its logical call.""" + self._close(logical_outcome="cancelled") + close_error = self._close_error + self._close_error = None + if close_error is not None: + raise close_error + + def _preserve_pending_provider_chunks(self) -> None: + """Switch a failed Relay stream to its undelivered provider chunks.""" + pending = [raw for _encoded, raw in self._raw_chunks] + self._raw_chunks.clear() + loop = self._loop + relay_stream = self._stream + self._loop = None + self._stream = iter(pending) + self._raw_stream_resource = None + self._accept_chunk = None + if loop is not None: + close = getattr(relay_stream, "aclose", None) + if callable(close): + + async def close_stream() -> None: + await close() + + try: + loop.run_until_complete(close_stream()) + except Exception: + logger.debug( + "Relay stream cleanup failed during provider fallback", + exc_info=True, + ) + loop.close() + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome="success") + self._logical = None + + def _close(self, *, logical_outcome: str) -> None: + if self._closed: + return + self._closed = True + loop = self._loop + self._loop = None + if loop is None: + resources = (self._stream, self._raw_stream_resource) + self._stream = None + self._raw_stream_resource = None + closed_ids: set[int] = set() + for resource in resources: + if resource is None or id(resource) in closed_ids: + continue + closed_ids.add(id(resource)) + close = getattr(resource, "close", None) + if callable(close): + try: + close() + except Exception as exc: + if self._close_error is None: + self._close_error = exc + logger.debug( + "Provider stream cleanup failed", + exc_info=True, + ) + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome=logical_outcome) + self._logical = None + return + close = getattr(self._stream, "aclose", None) + if callable(close): + + async def close_stream() -> None: + await close() + + try: + loop.run_until_complete(close_stream()) + except Exception as exc: + if self._close_error is None: + self._close_error = exc + if not self._defer_logical_completion: + _complete_logical(self._logical, outcome=logical_outcome) + self._logical = None + loop.close() + + def __del__(self) -> None: + self._close(logical_outcome="cancelled") + + +class AnthropicStreamAccumulator: + """Rebuild an Anthropic Message from post-intercept SSE events.""" + + def __init__(self) -> None: + self._message: dict[str, Any] = {} + self._blocks: dict[int, dict[str, Any]] = {} + + def observe(self, event: Any) -> None: + payload = _jsonable(event) + if not isinstance(payload, dict): + return + event_type = payload.get("type") + if event_type == "message_start": + message = payload.get("message") + if isinstance(message, dict): + for key in ("id", "type", "role", "model", "usage"): + if key in message: + self._message[key] = message[key] + return + if event_type == "content_block_start": + index = payload.get("index") + block = payload.get("content_block") + if isinstance(index, int) and isinstance(block, dict): + self._blocks[index] = dict(block) + return + if event_type == "content_block_delta": + index = payload.get("index") + delta = payload.get("delta") + if not isinstance(index, int) or not isinstance(delta, dict): + return + block = self._blocks.setdefault(index, {}) + delta_type = delta.get("type") + if delta_type == "text_delta": + block["text"] = str(block.get("text") or "") + str( + delta.get("text") or "" + ) + elif delta_type == "thinking_delta": + block["thinking"] = str(block.get("thinking") or "") + str( + delta.get("thinking") or "" + ) + elif delta_type == "signature_delta": + block["signature"] = str(block.get("signature") or "") + str( + delta.get("signature") or "" + ) + elif delta_type == "input_json_delta": + partial = str(block.pop("_partial_json", "")) + str( + delta.get("partial_json") or "" + ) + block["_partial_json"] = partial + elif delta_type == "citations_delta" and "citation" in delta: + block.setdefault("citations", []).append(delta["citation"]) + return + if event_type == "message_delta": + delta = payload.get("delta") + if isinstance(delta, dict): + for key in ("stop_reason", "stop_sequence"): + if key in delta: + self._message[key] = delta[key] + if "usage" in payload: + usage = payload["usage"] + current_usage = self._message.get("usage") + if isinstance(current_usage, dict) and isinstance(usage, dict): + self._message["usage"] = {**current_usage, **usage} + else: + self._message["usage"] = usage + + def finalize(self) -> dict[str, Any]: + blocks = [] + for index in sorted(self._blocks): + block = dict(self._blocks[index]) + partial = block.pop("_partial_json", None) + if partial is not None: + try: + block["input"] = json.loads(partial) + except (TypeError, ValueError): + block["input"] = partial + blocks.append(block) + return {**self._message, "content": blocks} + + def response(self, base: Any = None) -> Any: + """Return the attribute-shaped response consumed by Hermes.""" + assembled = self.finalize() + base_payload = _jsonable(base) + if not isinstance(base_payload, dict): + base_payload = {} + content = assembled.pop("content", []) + merged = {**base_payload, **assembled} + if content or "content" not in merged: + merged["content"] = content + return _namespace(merged) + + +def _logical_parent( + runtime: relay_runtime.RelayRuntime, + session: Any, + parent: Any, + metadata: dict[str, Any] | None, +) -> tuple[relay_runtime.RelayTurnContext, Any, str] | None: + turn = relay_runtime.active_turn(session.session_id) + request_id = str((metadata or {}).get("api_request_id") or "") + if turn is None or not request_id or turn.lease.host is not runtime: + return None + with turn.finalize_lock: + if turn.closed: + return None + with turn.logical_llm_lock: + handle = turn.logical_llm_calls.get(request_id) + if handle is None: + handle = runtime.run_in_session( + session, + runtime.relay.scope.push, + relay_runtime.LOGICAL_LLM_SCOPE, + runtime.relay.ScopeType.Function, + handle=parent, + input={}, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: runtime.runtime_id, + "hermes.call_role": str( + (metadata or {}).get("call_role") or "primary" + ), + }, + ) + turn.logical_llm_calls[request_id] = handle + return turn, handle, request_id + + +def _complete_logical( + logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, + *, + outcome: str, +) -> None: + if logical is None: + return + turn, handle, request_id = logical + lease = turn.lease + if not isinstance(lease.host, relay_runtime.RelayRuntime): + return + with turn.finalize_lock: + with turn.logical_llm_lock: + if turn.logical_llm_calls.get(request_id) is not handle: + return + if lease.session is None: + return + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + handle, + output={"outcome": outcome}, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + # The provider result is authoritative. Retain the handle so turn + # finalization can retry cleanup without changing that result. + logger.warning( + "Hermes Relay logical LLM finalization failed", + exc_info=True, + ) + return + with turn.logical_llm_lock: + if turn.logical_llm_calls.get(request_id) is handle: + turn.logical_llm_calls.pop(request_id, None) + + +def _recover_successful_callback( + raw_response: dict[str, Any], + *, + relay_error: BaseException, + callback_error: BaseException | None, + logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, + defer_logical_completion: bool, +) -> bool: + if ( + not isinstance(relay_error, Exception) + or callback_error is not None + or "value" not in raw_response + ): + return False + logger.warning( + "NeMo Relay LLM post-processing failed after provider success; " + "returning the provider response", + exc_info=True, + ) + if not defer_logical_completion: + _complete_logical(logical, outcome="success") + return True + + +def _is_cancellation(error: BaseException) -> bool: + return isinstance( + error, + (asyncio.CancelledError, InterruptedError, KeyboardInterrupt), + ) + + +def complete_logical_call(api_request_id: str, *, outcome: str) -> None: + """Complete the active turn's logical LLM call after caller validation.""" + turn = relay_runtime.active_turn() + if turn is None or not api_request_id: + return + with turn.logical_llm_lock: + handle = turn.logical_llm_calls.get(api_request_id) + if handle is not None: + _complete_logical((turn, handle, api_request_id), outcome=outcome) + + +def _provider_request( + original: dict[str, Any], + request: Any, + *, + relay_request_body: dict[str, Any], + codec_baseline_body: dict[str, Any] | None, + metadata: dict[str, Any] | None, +) -> dict[str, Any]: + content = getattr(request, "content", request) + if not isinstance(content, dict): + content = relay_request_body + if codec_baseline_body is None or _json_equal(content, relay_request_body): + final = dict(original) + else: + baseline = codec_baseline_body + intercepted = _provider_request_body(content, metadata) + final = dict(original) + # Typed codecs may not represent provider-specific fields. Overlay only + # values that changed from the codec-facing baseline so unrelated + # intercepts cannot delete or normalize unknown provider arguments. + for key in baseline.keys() | intercepted.keys(): + if key not in intercepted: + final.pop(key, None) + elif key not in baseline or not _json_equal( + intercepted[key], + baseline[key], + ): + final[key] = intercepted[key] + _restore_provider_message_extensions( + original, + final, + baseline=baseline, + intercepted=intercepted, + ) + headers = getattr(request, "headers", None) + if isinstance(headers, dict): + headers = { + key: value + for key, value in headers.items() + if str(key).lower() not in _RELAY_INTERNAL_PROVIDER_HEADERS + } + if headers: + final["extra_headers"] = { + **dict(final.get("extra_headers") or {}), + **headers, + } + return final + + +def _relay_request_body( + request: dict[str, Any], metadata: dict[str, Any] | None +) -> dict[str, Any]: + body = _jsonable(request) + if not isinstance(body, dict): + return {} + # The Responses SDK accepts ``tools=None`` as "no tools", while Relay's + # typed Responses codec correctly expects either an array or an absent + # field. Normalize only the codec-facing copy; the original provider + # request is restored when no interceptor changes it. + if str((metadata or {}).get("api_mode") or "") == "codex_responses": + body = dict(body) + if body.get("tools") is None: + body.pop("tools", None) + elif isinstance(body.get("tools"), list): + body["tools"] = [ + { + "type": "function", + "function": { + key: value + for key, value in tool.items() + if key != "type" + }, + } + if isinstance(tool, dict) + and tool.get("type") == "function" + and "function" not in tool + else tool + for tool in body["tools"] + ] + elif str((metadata or {}).get("api_mode") or "") == "chat_completions": + tools = body.get("tools") + if isinstance(tools, list): + body = dict(body) + body["tools"] = [ + {"type": "function", **tool} + if isinstance(tool, dict) + and "function" in tool + and "type" not in tool + else tool + for tool in tools + ] + return body + + +def _restore_provider_message_extensions( + original: dict[str, Any], + final: dict[str, Any], + *, + baseline: dict[str, Any], + intercepted: dict[str, Any], +) -> None: + """Restore provider wire fields that Relay's typed codec cannot represent.""" + original_messages = original.get("messages") + final_messages = final.get("messages") + baseline_messages = baseline.get("messages") + intercepted_messages = intercepted.get("messages") + if not all( + isinstance(messages, list) + for messages in ( + original_messages, + final_messages, + baseline_messages, + intercepted_messages, + ) + ): + return + if not ( + len(original_messages) + == len(final_messages) + == len(baseline_messages) + == len(intercepted_messages) + ): + return + for original_message, final_message, baseline_message, intercepted_message in zip( + original_messages, + final_messages, + baseline_messages, + intercepted_messages, + strict=True, + ): + if not all( + isinstance(message, dict) + for message in ( + original_message, + final_message, + baseline_message, + intercepted_message, + ) + ): + continue + for key in _PROVIDER_MESSAGE_EXTENSION_KEYS: + if ( + key in original_message + and key not in baseline_message + and key not in intercepted_message + and key not in final_message + ): + final_message[key] = original_message[key] + + +def _codec_round_trip_request_body( + relay: Any, + relay_request: Any, + *, + relay_request_body: dict[str, Any], + metadata: dict[str, Any] | None, +) -> dict[str, Any] | None: + """Return the codec-only request shape used to identify real rewrites.""" + codec = _codec(relay, metadata) + if codec is None: + return _provider_request_body(relay_request_body, metadata) + try: + annotated = codec.decode(relay_request) + encoded = codec.encode(annotated, relay_request) + content = getattr(encoded, "content", encoded) + if isinstance(content, dict): + return _provider_request_body(content, metadata) + except Exception: + logger.warning( + "NeMo Relay request codec baseline failed; ignoring request rewrites", + exc_info=True, + ) + return None + logger.warning( + "NeMo Relay request codec returned an unsupported baseline; " + "ignoring request rewrites" + ) + return None + + +def _provider_request_body( + content: dict[str, Any], metadata: dict[str, Any] | None +) -> dict[str, Any]: + body = dict(content) + if str((metadata or {}).get("api_mode") or "") != "codex_responses": + return body + tools = body.get("tools") + if not isinstance(tools, list): + return body + body["tools"] = [ + { + "type": "function", + **dict(tool["function"]), + } + if isinstance(tool, dict) + and tool.get("type") == "function" + and isinstance(tool.get("function"), dict) + else tool + for tool in tools + ] + return body + + +def _codec(relay: Any, metadata: dict[str, Any] | None) -> Any: + api_mode = str((metadata or {}).get("api_mode") or "") + codecs = getattr(relay, "codecs", None) + if codecs is None: + return None + if api_mode == "chat_completions": + codec = getattr(codecs, "OpenAIChatCodec", None) + elif api_mode == "anthropic_messages": + codec = getattr(codecs, "AnthropicMessagesCodec", None) + elif api_mode == "codex_responses": + codec = getattr(codecs, "OpenAIResponsesCodec", None) + else: + codec = None + return codec() if callable(codec) else None + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + model_dump = getattr(type(value), "model_dump", None) + if callable(model_dump): + try: + return _jsonable(value.model_dump(mode="json")) + except Exception: + pass + try: + attributes = { + str(key): item + for key, item in vars(value).items() + if not str(key).startswith("_") + } + except (TypeError, AttributeError): + return str(value) + return _jsonable(attributes) if attributes else str(value) + + +def _namespace(value: Any) -> Any: + if isinstance(value, dict): + return SimpleNamespace(**{ + str(key): _namespace(item) for key, item in value.items() + }) + if isinstance(value, list): + return [_namespace(item) for item in value] + return value + + +def _json_equal(left: Any, right: Any) -> bool: + try: + return json.dumps( + _jsonable(left), sort_keys=True, separators=(",", ":") + ) == json.dumps(_jsonable(right), sort_keys=True, separators=(",", ":")) + except (TypeError, ValueError): + return False + + +def _run_awaitable(value: Any) -> Any: + if not inspect.isawaitable(value): + return value + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(value) + raise RuntimeError( + "Synchronous Relay LLM execution cannot run on an event-loop thread" + ) diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py new file mode 100644 index 000000000000..533604791a86 --- /dev/null +++ b/agent/relay_runtime.py @@ -0,0 +1,1002 @@ +"""Profile-scoped NeMo Relay runtimes owned by the Hermes agent core.""" + +from __future__ import annotations + +import atexit +import asyncio +import contextvars +import importlib +import inspect +import logging +import threading +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable + +from hermes_constants import get_hermes_home + +logger = logging.getLogger(__name__) + +SESSION_SCOPE = "hermes.session" +TURN_SCOPE = "hermes.turn" +LOGICAL_LLM_SCOPE = "hermes.logical_llm_call" +RUNTIME_SCHEMA_KEY = "hermes.relay.schema_version" +RUNTIME_SCHEMA_VERSION = "hermes.relay.runtime.v1" +RUNTIME_INSTANCE_KEY = "hermes.relay.runtime_instance" +_PROFILE_KEY_CACHE: dict[str, str] = {} + + +@dataclass +class RelaySession: + """One isolated Relay scope stack owned by a Hermes session.""" + + session_id: str + parent_session_id: str = "" + lock: threading.RLock = field(default_factory=threading.RLock, repr=False) + closing: bool = False + handle: Any = None + context: contextvars.Context | None = None + + +class RelayRuntime: + """Own Relay session scopes independently of any exporter or plugin.""" + + def __init__(self, relay: Any = None, *, profile_key: str | None = None) -> None: + self.relay = relay or _load_nemo_relay() + self.profile_key = profile_key or current_profile_key() + self.runtime_id = uuid.uuid4().hex + self._sessions_lock = threading.RLock() + self._sessions: dict[str, RelaySession] = {} + self._subagent_parents: dict[str, str] = {} + self._subagent_parent_handles: dict[str, Any] = {} + self._execution_consumers_lock = threading.RLock() + self._execution_consumers: set[str] = set() + self._shutdown_registered = True + atexit.register(self.shutdown) + + def retain_managed_execution(self, consumer: str) -> None: + """Keep managed LLM and tool execution active for one consumer.""" + if not consumer: + raise ValueError("Relay managed-execution consumer must not be empty") + with self._execution_consumers_lock: + self._execution_consumers.add(consumer) + + def release_managed_execution(self, consumer: str) -> None: + """Release a consumer's managed-execution requirement.""" + with self._execution_consumers_lock: + self._execution_consumers.discard(consumer) + + def managed_execution_enabled(self) -> bool: + """Return whether a Hermes-managed consumer needs the Relay pipeline.""" + with self._execution_consumers_lock: + return bool(self._execution_consumers) + + def ensure_session( + self, + event: dict[str, Any], + *, + data: Any = None, + metadata: dict[str, Any] | None = None, + ) -> RelaySession | None: + """Return the existing session scope or create it once.""" + session_id = _session_id(event) + if not session_id: + return None + with self._sessions_lock: + session = self._sessions.get(session_id) + if session is None: + parent_session_id = self._subagent_parents.get(session_id, "") + session = RelaySession( + session_id=session_id, + parent_session_id=parent_session_id, + ) + self._sessions[session_id] = session + with session.lock: + if session.closing: + return None + if session.handle is None: + parent_handle = None + scope_metadata = { + **(metadata or {}), + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + } + if session.parent_session_id: + with self._sessions_lock: + parent_handle = self._subagent_parent_handles.get(session_id) + if parent_handle is None: + parent = self.ensure_session({ + "session_id": session.parent_session_id + }) + if parent is not None: + parent_handle = parent.handle + scope_metadata["nemo_relay_scope_role"] = "subagent" + context = contextvars.Context() + try: + session.handle = context.run( + self.relay.scope.push, + SESSION_SCOPE, + self.relay.ScopeType.Agent, + handle=parent_handle, + data=data, + input={}, + metadata=scope_metadata, + ) + except Exception: + session.context = None + raise + session.context = context + return session + + def register_subagent( + self, + event: dict[str, Any], + *, + metadata: dict[str, Any] | None = None, + ) -> RelaySession | None: + """Open a child Agent scope under its spawning turn when available.""" + parent_session_id = str(event.get("parent_session_id") or "") + child_session_id = str(event.get("child_session_id") or "") + if ( + not parent_session_id + or not child_session_id + or parent_session_id == child_session_id + ): + return None + parent = self.ensure_session({"session_id": parent_session_id}) + parent_handle = None if parent is None else parent.handle + turn = active_turn(parent_session_id) + if ( + turn is not None + and not turn.closed + and turn.handle is not None + and turn.lease.host is self + and turn.lease.session is not None + and turn.lease.session.session_id == parent_session_id + ): + parent_handle = turn.handle + with self._sessions_lock: + self._subagent_parents[child_session_id] = parent_session_id + if parent_handle is not None: + self._subagent_parent_handles[child_session_id] = parent_handle + return self.ensure_session( + {"session_id": child_session_id}, + metadata=metadata, + ) + + def unregister_subagent(self, event: dict[str, Any]) -> None: + """Close a delegated session and forget its parent relationship.""" + child_session_id = str(event.get("child_session_id") or "") + if not child_session_id: + return + self.close_session({"session_id": child_session_id}) + with self._sessions_lock: + self._subagent_parents.pop(child_session_id, None) + self._subagent_parent_handles.pop(child_session_id, None) + + def get_session(self, session_id: str) -> RelaySession | None: + """Return an active Hermes Relay session without creating one.""" + with self._sessions_lock: + session = self._sessions.get(str(session_id or "")) + if session is None: + return None + with session.lock: + return None if session.closing else session + + def get_session_handle(self, session_id: str) -> Any: + """Return the Relay parent handle for a Hermes session, if active.""" + session = self.get_session(session_id) + return None if session is None else session.handle + + def run_in_session( + self, + session: RelaySession, + callback: Callable[..., Any], + *args: Any, + allow_closing: bool = False, + **kwargs: Any, + ) -> Any: + """Run a Relay operation against a session's isolated scope stack.""" + with session.lock: + if session.closing and not allow_closing: + raise RuntimeError("Hermes Relay session is closing") + if session.context is None or session.handle is None: + raise RuntimeError("Hermes Relay session context is unavailable") + relay_context = session.context.copy() + + context = contextvars.copy_context() + for variable, value in relay_context.items(): + context.run(variable.set, value) + + def invoke() -> Any: + self.relay.get_scope_stack() + return callback(*args, **kwargs) + + # A copy permits a helper called by an existing Relay callback to + # re-enter the same logical session without re-entering Context. + return context.run(invoke) + + async def run_in_session_async( + self, + session: RelaySession, + callback: Callable[..., Any], + *args: Any, + allow_closing: bool = False, + **kwargs: Any, + ) -> Any: + """Create and await an operation inside the session's saved context.""" + with session.lock: + if session.closing and not allow_closing: + raise RuntimeError("Hermes Relay session is closing") + if session.context is None or session.handle is None: + raise RuntimeError("Hermes Relay session context is unavailable") + relay_context = session.context.copy() + + context = contextvars.copy_context() + for variable, value in relay_context.items(): + context.run(variable.set, value) + + async def invoke() -> Any: + self.relay.get_scope_stack() + result = callback(*args, **kwargs) + if inspect.isawaitable(result): + return await result + return result + + task = context.run(asyncio.create_task, invoke()) + return await task + + def emit_mark( + self, + name: str, + event: dict[str, Any], + *, + data: Any = None, + metadata: Any = None, + ) -> bool: + """Emit a mark parented to the Hermes session identified by ``event``.""" + session = self.ensure_session(event) + if session is None: + return False + self.run_in_session( + session, + self.relay.scope.event, + name, + handle=session.handle, + data=data, + metadata=metadata, + ) + return True + + def apply_tool_request_intercepts( + self, + *, + session_id: str, + tool_name: str, + args: dict[str, Any], + ) -> dict[str, Any]: + """Apply Relay request rewriting before Hermes authorizes a tool call.""" + if not self.managed_execution_enabled(): + return args + request_intercepts = getattr( + getattr(self.relay, "tools", None), + "request_intercepts", + None, + ) + if not callable(request_intercepts): + return args + session = self.ensure_session({"session_id": session_id}) + if session is None: + return args + result = self.run_in_session( + session, + request_intercepts, + tool_name, + args, + ) + return result if isinstance(result, dict) else args + + def close_session(self, event: dict[str, Any]) -> None: + """Close one session scope and remove it from the core registry.""" + session_id = _session_id(event) + with self._sessions_lock: + session = self._sessions.get(session_id) + if session is None: + with self._sessions_lock: + self._subagent_parents.pop(session_id, None) + self._subagent_parent_handles.pop(session_id, None) + return + failures: list[str] = [] + with session.lock: + if session.closing: + return + session.closing = True + if session.handle is not None: + try: + self.run_in_session( + session, + self.relay.scope.pop, + session.handle, + output={}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + }, + allow_closing=True, + ) + except Exception as exc: + failures.append(f"session scope close failed: {exc}") + try: + self.relay.subscribers.flush() + except Exception as exc: + failures.append(f"subscriber flush failed: {exc}") + with self._sessions_lock: + if self._sessions.get(session_id) is session: + self._sessions.pop(session_id, None) + self._subagent_parents.pop(session_id, None) + self._subagent_parent_handles.pop(session_id, None) + if failures: + logger.warning( + "Hermes Relay session %s closed with errors: %s", + session_id, + "; ".join(failures), + ) + + def shutdown(self) -> None: + """Close all core-owned Relay session scopes.""" + with self._sessions_lock: + session_ids = list(self._sessions) + for session_id in session_ids: + self._safe(self.close_session, {"session_id": session_id}) + if self._shutdown_registered: + try: + atexit.unregister(self.shutdown) + except Exception: + pass + self._shutdown_registered = False + + @staticmethod + def _safe(callback: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + try: + return callback(*args, **kwargs) + except Exception: + logger.warning("Hermes Relay runtime operation failed", exc_info=True) + return None + + +@dataclass(frozen=True) +class NoopRelayRuntime: + """Explicit reduced-capability host for platforms without Relay wheels.""" + + profile_key: str + reason: str + + @property + def available(self) -> bool: + return False + + def apply_tool_request_intercepts( + self, + *, + session_id: str, + tool_name: str, + args: dict[str, Any], + ) -> dict[str, Any]: + del session_id, tool_name + return args + + @staticmethod + def retain_managed_execution(consumer: str) -> None: + del consumer + + @staticmethod + def release_managed_execution(consumer: str) -> None: + del consumer + + @staticmethod + def managed_execution_enabled() -> bool: + return False + + def shutdown(self) -> None: + """No resources are allocated on unsupported platforms.""" + + +RelayHost = RelayRuntime | NoopRelayRuntime + + +class RelayHostRegistry: + """Own exactly one Relay host for each canonical Hermes profile.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._hosts: dict[str, RelayHost] = {} + + def for_profile( + self, + profile_key: str | None = None, + *, + create: bool = True, + ) -> RelayHost | None: + key = profile_key or current_profile_key() + host = self._hosts.get(key) + if host is not None or not create: + return host + with self._lock: + host = self._hosts.get(key) + if host is not None or not create: + return host + try: + host = RelayRuntime(profile_key=key) + except Exception as exc: + logger.warning( + "Hermes Relay runtime initialization failed", exc_info=True + ) + host = NoopRelayRuntime(profile_key=key, reason=str(exc)) + self._hosts[key] = host + return host + + def shutdown_profile(self, profile_key: str) -> None: + with self._lock: + host = self._hosts.pop(profile_key, None) + if host is not None: + host.shutdown() + + def shutdown_all(self) -> None: + with self._lock: + hosts = list(self._hosts.values()) + self._hosts.clear() + for host in hosts: + host.shutdown() + + +HOST_REGISTRY = RelayHostRegistry() + + +@dataclass +class ConversationLease: + """A resumable reference to one profile-scoped conversation scope.""" + + profile_key: str + session_id: str + platform: str + host: RelayHost + session: RelaySession | None + parent_session_id: str = "" + released: bool = False + + +@dataclass +class RelayTurnContext: + """Runtime-only context for one Hermes turn or top-level task.""" + + lease: ConversationLease + turn_id: str + task_id: str + handle: Any = None + logical_llm_calls: dict[str, Any] = field(default_factory=dict, repr=False) + logical_llm_lock: threading.RLock = field( + default_factory=threading.RLock, + repr=False, + ) + finalize_lock: threading.RLock = field( + default_factory=threading.RLock, + repr=False, + ) + _token: contextvars.Token[RelayTurnContext | None] | None = field( + default=None, + repr=False, + ) + _active_registered: bool = field(default=False, repr=False) + closed: bool = False + + +_CURRENT_TURN: contextvars.ContextVar[RelayTurnContext | None] = contextvars.ContextVar( + "hermes_relay_turn", default=None +) + + +class RelaySessionCoordinator: + """Own semantic conversation and turn lifetimes for Hermes core.""" + + def __init__(self, registry: RelayHostRegistry = HOST_REGISTRY) -> None: + self.registry = registry + self._initializer_lock = threading.RLock() + self._session_initializers: dict[ + str, + Callable[[RelayRuntime, dict[str, Any]], None], + ] = {} + self._active_turns_lock = threading.RLock() + self._active_turns: dict[tuple[str, str], set[int]] = {} + + def register_session_initializer( + self, + name: str, + callback: Callable[[RelayRuntime, dict[str, Any]], None], + ) -> None: + """Register idempotent profile/session preparation before scope creation.""" + with self._initializer_lock: + self._session_initializers[name] = callback + + def unregister_session_initializer(self, name: str) -> None: + """Remove a previously registered session initializer.""" + with self._initializer_lock: + self._session_initializers.pop(name, None) + + def _prepare_session( + self, + host: RelayRuntime, + context: dict[str, Any], + ) -> None: + with self._initializer_lock: + initializers = list(self._session_initializers.items()) + for name, callback in initializers: + try: + callback(host, context) + except Exception: + logger.warning( + "Hermes Relay session initializer failed: %s", + name, + exc_info=True, + ) + + def acquire_conversation( + self, + *, + profile_key: str, + session_id: str, + platform: str, + parent_session_id: str = "", + model: str = "", + ) -> ConversationLease: + host = self.registry.for_profile(profile_key) + if host is None: + host = NoopRelayRuntime(profile_key, "Relay host creation was disabled") + session = None + if isinstance(host, RelayRuntime): + try: + session_context = { + "profile_key": profile_key, + "session_id": session_id, + "platform": platform, + "parent_session_id": parent_session_id, + "model": model, + } + self._prepare_session(host, session_context) + metadata = {"hermes.execution_surface": platform or "unknown"} + if parent_session_id and parent_session_id != session_id: + session = host.register_subagent( + { + "parent_session_id": parent_session_id, + "child_session_id": session_id, + }, + metadata=metadata, + ) + else: + session = host.ensure_session( + {"session_id": session_id}, + metadata=metadata, + ) + except Exception: + logger.warning( + "Hermes Relay conversation initialization failed", + exc_info=True, + ) + return ConversationLease( + profile_key=profile_key, + session_id=session_id, + platform=platform, + host=host, + session=session, + parent_session_id=parent_session_id, + ) + + def begin_turn( + self, + lease: ConversationLease, + *, + turn_id: str, + task_id: str, + ) -> RelayTurnContext: + if lease.released: + raise RuntimeError("Hermes Relay conversation lease is released") + turn = RelayTurnContext(lease=lease, turn_id=turn_id, task_id=task_id) + if isinstance(lease.host, RelayRuntime) and lease.session is not None: + try: + turn.handle = lease.host.run_in_session( + lease.session, + lease.host.relay.scope.push, + TURN_SCOPE, + lease.host.relay.ScopeType.Function, + handle=lease.session.handle, + input={}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + "hermes.execution_surface": lease.platform or "unknown", + }, + ) + except Exception: + logger.warning("Hermes Relay turn initialization failed", exc_info=True) + turn._token = _CURRENT_TURN.set(turn) + key = (lease.profile_key, lease.session_id) + with self._active_turns_lock: + self._active_turns.setdefault(key, set()).add(id(turn)) + turn._active_registered = True + return turn + + def end_turn( + self, + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + with turn.finalize_lock: + if turn.closed: + self._reset_turn_context(turn) + return + turn.closed = True + lease = turn.lease + try: + if isinstance(lease.host, RelayRuntime) and lease.session is not None: + self._finish_logical_calls(turn, outcome=outcome) + if turn.handle is not None: + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + turn.handle, + output={"outcome": outcome}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + logger.warning( + "Hermes Relay turn finalization failed", exc_info=True + ) + finally: + try: + # Delegated agents own one turn. Close their conversation + # while the active-turn guard is still held so a parent + # timeout fallback cannot race this terminal boundary. + if ( + lease.parent_session_id + and isinstance(lease.host, RelayRuntime) + ): + lease.host.unregister_subagent({ + "child_session_id": lease.session_id + }) + except Exception: + logger.warning( + "Hermes Relay child conversation finalization failed", + exc_info=True, + ) + finally: + self._unregister_active_turn(turn) + self._reset_turn_context(turn) + + def has_active_turn(self, *, profile_key: str, session_id: str) -> bool: + """Return whether a turn is still running for one profile/session.""" + key = (profile_key, session_id) + with self._active_turns_lock: + return bool(self._active_turns.get(key)) + + def _unregister_active_turn(self, turn: RelayTurnContext) -> None: + if not turn._active_registered: + return + key = (turn.lease.profile_key, turn.lease.session_id) + with self._active_turns_lock: + active = self._active_turns.get(key) + if active is not None: + active.discard(id(turn)) + if not active: + self._active_turns.pop(key, None) + turn._active_registered = False + + def _reset_active_turns_for_tests(self) -> None: + with self._active_turns_lock: + self._active_turns.clear() + + def finish_logical_calls( + self, + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + """Close logical LLM children before sibling task aggregation scopes.""" + with turn.finalize_lock: + if turn.closed: + return + self._finish_logical_calls(turn, outcome=outcome) + + @staticmethod + def _finish_logical_calls( + turn: RelayTurnContext, + *, + outcome: str, + ) -> None: + lease = turn.lease + if not isinstance(lease.host, RelayRuntime) or lease.session is None: + return + with turn.logical_llm_lock: + logical_calls = list(turn.logical_llm_calls.items()) + turn.logical_llm_calls.clear() + for index in range(len(logical_calls) - 1, -1, -1): + request_id, logical_handle = logical_calls[index] + try: + lease.host.run_in_session( + lease.session, + lease.host.relay.scope.pop, + logical_handle, + output={"outcome": outcome}, + metadata={ + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: lease.host.runtime_id, + }, + ) + except Exception: + with turn.logical_llm_lock: + # Relay scopes are stack-owned. If the newest remaining + # handle cannot close, older handles cannot close safely + # either, so retain the unclosed prefix for diagnostics. + for pending_request_id, pending_handle in logical_calls[ + : index + 1 + ]: + turn.logical_llm_calls.setdefault( + pending_request_id, + pending_handle, + ) + logger.warning( + "Hermes Relay logical LLM finalization failed", + exc_info=True, + ) + break + + @staticmethod + def _reset_turn_context(turn: RelayTurnContext) -> None: + """Reset the originating ContextVar token when called in that context.""" + if turn._token is None: + return + try: + _CURRENT_TURN.reset(turn._token) + except ValueError: + # A copied async/thread context may own terminal cleanup. Keep the + # token so the originating context can clear its stale reference. + return + turn._token = None + + @staticmethod + def release_conversation(lease: ConversationLease) -> None: + """Release a caller lease without closing a resumable conversation.""" + lease.released = True + + def finalize_conversation( + self, + *, + profile_key: str, + session_id: str, + ) -> None: + host = self.registry.for_profile(profile_key, create=False) + if isinstance(host, RelayRuntime): + host.close_session({"session_id": session_id}) + + def shutdown_profile(self, profile_key: str) -> None: + self.registry.shutdown_profile(profile_key) + + +SESSION_COORDINATOR = RelaySessionCoordinator() + + +def current_turn() -> RelayTurnContext | None: + """Return the turn context inherited by current async and thread work.""" + return _CURRENT_TURN.get() + + +def active_turn(session_id: str | None = None) -> RelayTurnContext | None: + """Return a live turn only when it belongs to the active profile/session.""" + turn = current_turn() + if turn is None or turn.closed or turn.lease.released: + return None + if turn.lease.profile_key != current_profile_key(): + return None + if session_id is not None and turn.lease.session_id != session_id: + return None + if isinstance(turn.lease.host, RelayRuntime): + if turn.lease.session is None: + return None + if turn.lease.host.get_session(turn.lease.session_id) is not turn.lease.session: + return None + return turn + + +def resolve_execution_context( + session_id: str, +) -> tuple[RelayRuntime | None, RelaySession | None, Any]: + """Resolve one active turn/session parent for managed Relay execution.""" + turn = active_turn(session_id) + if ( + turn is not None + and isinstance(turn.lease.host, RelayRuntime) + and turn.lease.session is not None + ): + session = turn.lease.session + return turn.lease.host, session, turn.handle or session.handle + # Managed-execution consumers create and retain the profile host before + # reaching an out-of-turn adapter. Do not initialize Relay for the default + # no-consumer path. + runtime = get_runtime(create=False) + if runtime is None: + return None, None, None + if not runtime.managed_execution_enabled(): + return None, None, None + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + return runtime, session, None if session is None else session.handle + + +def emit_mark( + name: str, + *, + session_id: str, + data: Any = None, + metadata: Any = None, +) -> bool: + """Emit a fail-open Relay mark under a Hermes session.""" + runtime = get_runtime(create=False) + if runtime is None: + return False + try: + return runtime.emit_mark( + name, + {"session_id": session_id}, + data=data, + metadata=metadata, + ) + except Exception: + logger.warning("Hermes Relay mark failed: %s", name, exc_info=True) + return False + + +def apply_tool_request_intercepts( + *, + session_id: str, + tool_name: str, + args: dict[str, Any], +) -> dict[str, Any]: + """Return Relay-rewritten arguments at Hermes's authorization boundary.""" + if not session_id: + return args + runtime = get_runtime(create=False) + if runtime is None: + return args + return runtime.apply_tool_request_intercepts( + session_id=session_id, + tool_name=tool_name, + args=args, + ) + + +def ensure_session(*, session_id: str, **context: Any) -> RelaySession | None: + """Create or return the shared Relay session used by Hermes core.""" + runtime = get_runtime() + if runtime is None: + return None + try: + return runtime.ensure_session({"session_id": session_id, **context}) + except Exception: + logger.warning("Hermes Relay session initialization failed", exc_info=True) + return None + + +def run_in_session( + session_id: str, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> Any: + """Run a scope, LLM, or tool API against a shared Hermes session.""" + runtime = get_runtime() + if runtime is None: + raise RuntimeError("Hermes Relay runtime is unavailable") + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay session is unavailable") + return runtime.run_in_session(session, callback, *args, **kwargs) + + +async def run_in_session_async( + session_id: str, + callback: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> Any: + """Await a Relay operation inside a shared Hermes session context.""" + runtime = get_runtime() + if runtime is None: + raise RuntimeError("Hermes Relay runtime is unavailable") + session = runtime.get_session(session_id) + if session is None: + session = runtime.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay session is unavailable") + return await runtime.run_in_session_async(session, callback, *args, **kwargs) + + +def get_session_handle(session_id: str) -> Any: + """Return the shared Relay handle for direct core instrumentation.""" + runtime = get_runtime(create=False) + return None if runtime is None else runtime.get_session_handle(session_id) + + +def _is_relay_wrapped_callback_error( + relay_error: BaseException, + callback_error: BaseException, +) -> bool: + """Match Relay's native callback wrapper without masking policy errors.""" + if relay_error is callback_error: + return True + if not isinstance(relay_error, RuntimeError): + return False + callback_type = callback_error.__class__ + type_names = { + callback_type.__name__, + callback_type.__qualname__, + f"{callback_type.__module__}.{callback_type.__qualname__}", + } + message = str(relay_error) + return any( + message.startswith(f"internal error: {type_name}: {callback_error}") + for type_name in type_names + ) + + +def get_runtime( + *, + create: bool = True, + profile_key: str | None = None, +) -> RelayRuntime | None: + """Return the Relay host for the active Hermes profile.""" + host = HOST_REGISTRY.for_profile(profile_key, create=create) + return host if isinstance(host, RelayRuntime) else None + + +def get_host( + *, + create: bool = True, + profile_key: str | None = None, +) -> RelayHost | None: + """Return the explicit real or reduced-capability host for a profile.""" + return HOST_REGISTRY.for_profile(profile_key, create=create) + + +def current_profile_key() -> str: + """Return the canonical profile identity used for runtime isolation.""" + home = get_hermes_home().expanduser() + if not home.is_absolute(): + return str(home.resolve()) + raw = str(home) + cached = _PROFILE_KEY_CACHE.get(raw) + if cached is not None: + return cached + resolved = str(home.resolve()) + return _PROFILE_KEY_CACHE.setdefault(raw, resolved) + + +def _load_nemo_relay() -> Any: + """Load the binding only when a producer or consumer needs Relay.""" + return importlib.import_module("nemo_relay") + + +def _session_id(event: dict[str, Any]) -> str: + return str(event.get("session_id") or "") + + +def _reset_for_tests() -> None: + """Reset all profile-scoped Relay hosts for isolated tests.""" + SESSION_COORDINATOR._reset_active_turns_for_tests() + HOST_REGISTRY.shutdown_all() + _PROFILE_KEY_CACHE.clear() diff --git a/agent/relay_tools.py b/agent/relay_tools.py new file mode 100644 index 000000000000..5023df1bcf92 --- /dev/null +++ b/agent/relay_tools.py @@ -0,0 +1,123 @@ +"""Core NeMo Relay adapter for Hermes tool execution.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import logging +from collections.abc import Callable +from typing import Any + +from agent import relay_runtime + +logger = logging.getLogger(__name__) + + +def execute( + tool_name: str, + args: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + session_id: str, + metadata: dict[str, Any] | None = None, +) -> tuple[Any, dict[str, Any]]: + """Run one tool call through Relay and return its final arguments.""" + runtime, session, parent = relay_runtime.resolve_execution_context(session_id) + if runtime is None or session is None or not runtime.managed_execution_enabled(): + return callback(args), args + + observed_args = args + raw_result: dict[str, Any] = {} + callback_error: BaseException | None = None + callback_context = contextvars.copy_context() + + def invoke(next_args: Any) -> Any: + nonlocal callback_error, observed_args + observed_args = next_args if isinstance(next_args, dict) else args + try: + result = callback_context.copy().run(callback, observed_args) + except BaseException as exc: + callback_error = exc + raise + raw_result["value"] = result + raw_result["json"] = _jsonable(result) + return raw_result["json"] + + try: + managed = _run_awaitable( + runtime.run_in_session_async( + session, + runtime.relay.tools.execute, + tool_name, + _jsonable(args), + invoke, + handle=parent, + metadata=_jsonable(metadata or {}), + ) + ) + except BaseException as exc: + if ( + callback_error is not None + and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error) + ): + raise callback_error + if ( + isinstance(exc, Exception) + and callback_error is None + and "value" in raw_result + ): + logger.warning( + "NeMo Relay tool post-processing failed after dispatch success; " + "returning the Hermes tool result", + exc_info=True, + ) + return raw_result["value"], observed_args + raise + + if "value" in raw_result and _json_equal(managed, raw_result["json"]): + return raw_result["value"], observed_args + if isinstance(managed, str): + return managed, observed_args + return json.dumps(_jsonable(managed), ensure_ascii=False), observed_args + + +def _jsonable(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set)): + return [_jsonable(item) for item in value] + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return _jsonable(model_dump(mode="json")) + except Exception: + pass + try: + return _jsonable(vars(value)) + except (TypeError, AttributeError): + return str(value) + + +def _json_equal(left: Any, right: Any) -> bool: + try: + return json.dumps( + _jsonable(left), sort_keys=True, separators=(",", ":") + ) == json.dumps(_jsonable(right), sort_keys=True, separators=(",", ":")) + except (TypeError, ValueError): + return left == right + + +def _run_awaitable(value: Any) -> Any: + if not inspect.isawaitable(value): + return value + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(value) + raise RuntimeError( + "Synchronous Hermes Relay tool execution cannot run on an active event-loop thread" + ) diff --git a/agent/retry_utils.py b/agent/retry_utils.py index c4971122394f..58c6231b6006 100644 --- a/agent/retry_utils.py +++ b/agent/retry_utils.py @@ -8,7 +8,9 @@ import random import threading import time -from typing import Any +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from typing import Any, Optional # Monotonic counter for jitter seed uniqueness within the same process. # Protected by a lock to avoid race conditions in concurrent retry paths @@ -33,6 +35,58 @@ _ZAI_CODING_OVERLOAD_SHORT_ATTEMPTS = 3 +def parse_retry_after_seconds(value_or_headers: Any) -> Optional[float]: + """Parse a ``Retry-After`` value into non-negative seconds. + + Accepts either a raw header value (numeric string / HTTP-date / number) + or a headers mapping, in which case the ``Retry-After`` key is looked up + case-insensitively (``.get`` on dict-like objects tries both common + casings; real HTTP header containers like httpx/requests are already + case-insensitive). + + Returns: + Seconds as a ``float`` (negative deltas clamped to ``0.0``), or + ``None`` when the header is absent or unparseable. + """ + raw = value_or_headers + if raw is not None and not isinstance(raw, (str, int, float)): + # Looks like a headers mapping — pull the header out of it. + getter = getattr(raw, "get", None) + if callable(getter): + try: + value = getter("Retry-After") + if value is None: + value = getter("retry-after") + except Exception: + return None + raw = value + else: + return None + if raw is None: + return None + if isinstance(raw, bool): + return None + if isinstance(raw, (int, float)): + return max(0.0, float(raw)) + text = str(raw).strip() + if not text: + return None + try: + return max(0.0, float(text)) + except (TypeError, ValueError): + pass + # HTTP-date form (RFC 7231): seconds until that instant, clamped at 0. + try: + when = parsedate_to_datetime(text) + except (TypeError, ValueError): + return None + if when is None: + return None + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + return max(0.0, (when - datetime.now(timezone.utc)).total_seconds()) + + def jittered_backoff( attempt: int, *, diff --git a/agent/runtime_cwd.py b/agent/runtime_cwd.py index 87c5f00e1832..712e38ed137e 100644 --- a/agent/runtime_cwd.py +++ b/agent/runtime_cwd.py @@ -2,10 +2,9 @@ `TERMINAL_CWD` is the runtime carrier for the configured working directory (design #19214/#19242: `terminal.cwd` is bridged once to `TERMINAL_CWD` at -gateway/cron/CLI startup). For the local CLI, an explicit `terminal.cwd` is -respected; placeholders such as `.`/`auto` resolve to the launch dir. Reading it -in one place keeps the system prompt, the tool surfaces, and context-file -discovery agreeing on where the agent lives. +gateway/cron startup). The local-CLI backend deliberately leaves it unset and +relies on the launch dir. Reading it in one place keeps the system prompt, the +tool surfaces, and context-file discovery agreeing on where the agent lives. Multi-session gateways can pin a logical cwd via the `_SESSION_CWD` contextvar; CLI/cron fall through to `TERMINAL_CWD`/launch cwd. diff --git a/agent/secret_scope.py b/agent/secret_scope.py index 974dc5e71b11..919fe3e27bd3 100644 --- a/agent/secret_scope.py +++ b/agent/secret_scope.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from contextvars import ContextVar, Token from pathlib import Path from typing import Dict, Mapping, Optional @@ -105,6 +106,14 @@ def current_secret_scope() -> Optional[Mapping[str, str]]: "VIRTUAL_ENV", "PYTHONPATH", "SSL_CERT_FILE", # Kanban paths (per-board, not per-profile-secret) "HERMES_KANBAN_DB", "HERMES_KANBAN_WORKSPACES_ROOT", "HERMES_KANBAN_BOARD", + # API-server LISTENER settings — deployment config (Docker compose + # ``environment:`` block, systemd ``Environment=``), not profile secrets. + # The scoped runner reload (#64674) must keep seeing them or container + # deployments silently lose the api_server platform (#69379). NOTE: + # API_SERVER_KEY is deliberately NOT here — it IS a credential and stays + # profile-scoped. + "API_SERVER_ENABLED", "API_SERVER_HOST", "API_SERVER_PORT", + "API_SERVER_CORS_ORIGINS", }) _GLOBAL_ENV_PREFIXES = ( "HERMES_KANBAN_", @@ -177,20 +186,72 @@ def get_secret(name: str, default: Optional[str] = None) -> Optional[str]: return val if val is not None else default +def _strip_inline_comment(value: str) -> str: + """Strip a dotenv-style inline comment from a raw ``.env`` value. + + Mirrors python-dotenv (1.2.2) semantics, verified empirically: + + - Quoted values: scan for the matching close quote + (backslash-escape-aware for double quotes, since ``save_env_value`` + writes ``\\"``/``\\\\`` escapes). Everything through the close quote is + kept; a trailing ``# ...`` remainder after it is discarded, so + ``KEY="has # inside" # trailing`` yields ``has # inside``. Non-comment + trailing junk leaves the value untouched (lenient, unlike dotenv's + hard parse error). + - Unquoted values: truncate only at a ``#`` PRECEDED BY WHITESPACE, so + ``KEY=foo#bar`` keeps ``foo#bar`` while ``KEY=value # comment`` keeps + ``value``. A value that *starts* with ``#`` (``KEY=#leading``) is kept. + """ + value = value.strip() + if not value: + return value + quote = value[0] + if quote in ("'", '"'): + i = 1 + while i < len(value): + ch = value[i] + if quote == '"' and ch == "\\": + i += 2 # skip the escaped character + continue + if ch == quote: + remainder = value[i + 1:].lstrip() + if remainder.startswith("#"): + return value[: i + 1] + return value + i += 1 + return value # unterminated quote: leave as-is + return re.split(r"\s+#", value, maxsplit=1)[0].strip() + + def load_env_file(env_path: Path) -> Dict[str, str]: """Parse a ``.env`` file into a plain dict WITHOUT touching ``os.environ``. Used to load a profile's secrets into an isolated mapping for - ``set_secret_scope``. Mirrors python-dotenv's basic parsing (KEY=VALUE, - ``export`` prefix, ``#`` comments, optional matching quotes) but never - mutates the process environment — that isolation is the whole point. + ``set_secret_scope``. Parses the small KEY=VALUE subset Hermes writes + itself (``export`` prefix, ``#`` comments — full-line and + dotenv-compatible inline, matching quotes with the + writer's ``\\"``/``\\\\`` escapes reversed — the same semantics as + ``hermes_cli.config._parse_env_value``) but never mutates the process + environment — that isolation is the whole point. + + Encoding is ``utf-8-sig`` so a leading UTF-8 BOM (Windows Notepad / + PowerShell ``Set-Content -Encoding UTF8``) does not prefix the first + key as ``\\ufeffNAME`` and make ``get_secret('NAME')`` miss under scope. """ secrets: Dict[str, str] = {} try: - text = env_path.read_text(encoding="utf-8") + text = env_path.read_text(encoding="utf-8-sig") except (FileNotFoundError, OSError, UnicodeDecodeError): return secrets + # Parse values with the canonical Hermes parser: save_env_value + # escapes " and \ inside double quotes, and every other reader + # (load_env, python-dotenv) reverses those escapes. Stripping only + # the outer quotes here would corrupt credentials containing " + # or \ — they work interactively but fail in scoped (cron / + # multiplex) resolution. + from hermes_cli.config import _parse_env_value + for raw in text.splitlines(): line = raw.strip() if not line or line.startswith("#"): @@ -203,10 +264,7 @@ def load_env_file(env_path: Path) -> Dict[str, str]: key = key.strip() if not key: continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - secrets[key] = value + secrets[key] = _parse_env_value(_strip_inline_comment(value)) return secrets @@ -222,14 +280,8 @@ def build_profile_secret_scope(hermes_home: Path) -> Dict[str, str]: secrets = load_env_file(home / ".env") try: - from hermes_cli.env_loader import load_profile_secret_source_values - - # 1Password service-account bootstrap material may live in .op.env; - # expose it only to the isolated source fetch. User .env wins when the - # same key exists in both, matching load_hermes_dotenv(). - source_env = load_env_file(home / ".op.env") - source_env.update(secrets) - external_secrets = load_profile_secret_source_values(home, source_env) + from hermes_cli.env_loader import get_secret_source_values + external_secrets = get_secret_source_values(home) except Exception: external_secrets = {} diff --git a/agent/secret_sources/_cache.py b/agent/secret_sources/_cache.py index 03bd4eb70958..64e1e5e13ca0 100644 --- a/agent/secret_sources/_cache.py +++ b/agent/secret_sources/_cache.py @@ -82,7 +82,9 @@ def resolve_cache_home(home_path: Optional[Path] = None) -> Path: (and tests that don't thread a home through) working. """ if home_path is None: - home_path = Path(os.getenv("HERMES_HOME", Path.home() / ".hermes")) + from hermes_constants import get_hermes_home + + home_path = get_hermes_home() return home_path diff --git a/agent/secret_sources/base.py b/agent/secret_sources/base.py index 277e25945c6d..a7bc345baa2c 100644 --- a/agent/secret_sources/base.py +++ b/agent/secret_sources/base.py @@ -39,51 +39,43 @@ import os import re import subprocess +from contextvars import ContextVar, Token from abc import ABC, abstractmethod -from contextlib import contextmanager -from contextvars import ContextVar from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Dict, FrozenSet, Iterator, List, Mapping, Optional, Sequence +from typing import Dict, FrozenSet, List, MutableMapping, Optional, Sequence # Bump ONLY for breaking changes to the required contract surface # (abstract-method signatures, FetchResult required fields). Additive # optional hooks must ship with defaults and must NOT bump this. SECRET_SOURCE_API_VERSION = 1 -# Timeout the orchestrator enforces around fetch() when the source's -# config section doesn't override it. Generous because a first run may -# include a one-time CLI binary auto-install (e.g. bws download+verify). -DEFAULT_FETCH_TIMEOUT_SECONDS = 120.0 +_SOURCE_ENVIRONMENT: ContextVar[Optional[MutableMapping[str, str]]] +_SOURCE_ENVIRONMENT = ContextVar("hermes_secret_source_environment", default=None) -# Default timeout for run_secret_cli() subprocess invocations. -DEFAULT_CLI_TIMEOUT_SECONDS = 30.0 -_SOURCE_ENVIRON: ContextVar[Optional[Mapping[str, str]]] = ContextVar( - "_SOURCE_ENVIRON", default=None -) +def set_source_environment(environ: MutableMapping[str, str]) -> Token: + """Install a per-fetch environment view without changing ``os.environ``.""" + return _SOURCE_ENVIRONMENT.set(environ) -def source_environ() -> Mapping[str, str]: - """Return the environment visible to the active source fetch. +def reset_source_environment(token: Token) -> None: + _SOURCE_ENVIRONMENT.reset(token) - Registry-driven profile fetches install an isolated mapping here so - backends can resolve bootstrap credentials without reading or mutating - another profile's process-global environment. - """ - scoped = _SOURCE_ENVIRON.get() - return scoped if scoped is not None else os.environ +def get_source_environment() -> MutableMapping[str, str]: + """Return the active per-fetch environment, or the process environment.""" + environ = _SOURCE_ENVIRONMENT.get() + return environ if environ is not None else os.environ -@contextmanager -def use_source_environ(environ: Mapping[str, str]) -> Iterator[None]: - """Install an isolated source environment for one fetch worker.""" - token = _SOURCE_ENVIRON.set(environ) - try: - yield - finally: - _SOURCE_ENVIRON.reset(token) +# Timeout the orchestrator enforces around fetch() when the source's +# config section doesn't override it. Generous because a first run may +# include a one-time CLI binary auto-install (e.g. bws download+verify). +DEFAULT_FETCH_TIMEOUT_SECONDS = 120.0 + +# Default timeout for run_secret_cli() subprocess invocations. +DEFAULT_CLI_TIMEOUT_SECONDS = 30.0 class ErrorKind(str, Enum): @@ -174,9 +166,6 @@ def fetch(self, cfg: dict, home_path: Path) -> FetchResult: ``cfg`` is the source's raw config section (``secrets.``) from config.yaml — treat every field defensively, the section may be malformed. ``home_path`` is the resolved HERMES_HOME. - Read bootstrap/process values through :func:`source_environ`, not - directly from ``os.environ``; multiplexed profile fetches install an - isolated mapping there to prevent cross-profile credential reads. """ # -- optional hooks (defaults are correct for most sources) ------------ @@ -269,6 +258,10 @@ def remediation(self, kind: Optional["ErrorKind"], cfg: dict) -> str: # ANSI CSI/OSC escape sequences — helper-CLI stderr often carries color # codes that must not reach Hermes' own startup output. +# NOTE: intentionally NOT migrated to tools.ansi_strip.strip_ansi — the +# optional terminator here (``(?:\x07|\x1b\\)?``) also strips *unterminated* +# OSC sequences (common when a CLI is killed mid-write), which strip_ansi +# leaves untouched. strip_ansi is not a superset of this regex. _ANSI_RE = re.compile(r"\x1b(?:\[[0-9;?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)?)") @@ -313,7 +306,7 @@ def run_secret_cli( "LANG", "LC_ALL", "XDG_CONFIG_HOME", "XDG_DATA_HOME") env: Dict[str, str] = {} for key in (*base_keep, *allow_env): - val = source_environ().get(key) + val = os.environ.get(key) if val is not None: env[key] = val if extra_env: @@ -325,7 +318,7 @@ def run_secret_cli( list(argv), env=env, capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", timeout=timeout, stdin=subprocess.DEVNULL, ) diff --git a/agent/secret_sources/bitwarden.py b/agent/secret_sources/bitwarden.py index d9c8cd59b7f4..357f69cc6fb1 100644 --- a/agent/secret_sources/bitwarden.py +++ b/agent/secret_sources/bitwarden.py @@ -57,7 +57,8 @@ FetchResult, is_valid_env_name as _is_valid_env_name, ) -from agent.secret_sources.base import ErrorKind, SecretSource, source_environ +from agent.secret_sources.base import ErrorKind, SecretSource +from agent.secret_sources.base import get_source_environment logger = logging.getLogger(__name__) @@ -200,7 +201,7 @@ def _platform_asset_name() -> str: res = subprocess.run( ["ldd", "--version"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=2, stdin=subprocess.DEVNULL, ) @@ -667,7 +668,15 @@ def _run_bws_list( bws: Path, access_token: str, project_id: str, server_url: str = "" ) -> Tuple[Dict[str, str], List[str]]: cmd = [str(bws), "secret", "list", project_id, "--output", "json"] - env = dict(source_environ()) + # bws child intentionally receives the access token. Under a profile-local + # fetch it must not inherit sibling credentials from process-global env. + source_env = get_source_environment() + if source_env is os.environ: + from tools.environments.local import build_subprocess_env + + env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False) + else: + env = dict(source_env) env["BWS_ACCESS_TOKEN"] = access_token # Make sure we're not echoing telemetry / colour codes into json. env.setdefault("NO_COLOR", "1") @@ -684,7 +693,7 @@ def _run_bws_list( cmd, env=env, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=_BWS_RUN_TIMEOUT, stdin=subprocess.DEVNULL, ) @@ -773,7 +782,7 @@ def apply_bitwarden_secrets( if not enabled: return result - access_token = source_environ().get(access_token_env, "").strip() + access_token = os.environ.get(access_token_env, "").strip() if not access_token: result.error = ( f"secrets.bitwarden.enabled is true but {access_token_env} is " @@ -905,7 +914,7 @@ def fetch(self, cfg: dict, home_path: Path) -> FetchResult: result = FetchResult() access_token_env = str(cfg.get("access_token_env") or "BWS_ACCESS_TOKEN") - access_token = source_environ().get(access_token_env, "").strip() + access_token = get_source_environment().get(access_token_env, "").strip() if not access_token: result.error = ( f"secrets.bitwarden.enabled is true but {access_token_env} is " diff --git a/agent/secret_sources/command.py b/agent/secret_sources/command.py index 356a47a0bcbd..083164841899 100644 --- a/agent/secret_sources/command.py +++ b/agent/secret_sources/command.py @@ -43,7 +43,8 @@ # Reuse the exact result shape the bitwarden source returns so # hermes_cli.env_loader can consume both providers identically. -from agent.secret_sources.base import ErrorKind, SecretSource, source_environ +from agent.secret_sources.base import ErrorKind, SecretSource +from agent.secret_sources.base import get_source_environment from agent.secret_sources.bitwarden import FetchResult __all__ = [ @@ -179,7 +180,19 @@ def _run_helper( ) return None - env = dict(source_environ()) + # User-configured secret-helper command: runs with the user's full shell + # env by design (it may need any credential to resolve the secret). + source_env = get_source_environment() + if source_env is os.environ: + # Legacy single-profile startup intentionally preserves the existing + # helper contract, which may rely on the user's full environment. + from tools.environments.local import build_subprocess_env + env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False) + else: + # A multiplex profile must never inherit sibling secrets from the + # process-global environment. hydrate_profile_secret_sources seeds + # only global-safe values plus this profile's own .env. + env = dict(source_env) env["HERMES_SECRET_KEY"] = secret_key try: @@ -265,10 +278,7 @@ def _parse_dotenv_map(stdout: str) -> Dict[str, str]: m = _ENV_LINE.match(line) if not m: continue - value = unquote_dotenv_value(m.group(2)) - if value.strip() == "": - continue - out[m.group(1)] = value + out[m.group(1)] = unquote_dotenv_value(m.group(2)) return out diff --git a/agent/secret_sources/onepassword.py b/agent/secret_sources/onepassword.py index 34a10624898d..3aef02df5b8a 100644 --- a/agent/secret_sources/onepassword.py +++ b/agent/secret_sources/onepassword.py @@ -42,7 +42,6 @@ import hashlib import logging import os -import re import shutil import subprocess import time @@ -55,7 +54,8 @@ FetchResult, is_valid_env_name, ) -from agent.secret_sources.base import ErrorKind, SecretSource, source_environ +from agent.secret_sources.base import ErrorKind, SecretSource +from agent.secret_sources.base import get_source_environment logger = logging.getLogger(__name__) @@ -73,10 +73,10 @@ # looks for. _DEFAULT_TOKEN_ENV = "OP_SERVICE_ACCOUNT_TOKEN" -# Strip whole ANSI CSI sequences (colour, cursor moves, line erases) from any -# `op` diagnostic we surface — not just the lone ESC byte — so a control -# sequence can't reposition the cursor or hide text after a redaction marker. -_ANSI_CSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") +# ANSI stripping for `op` diagnostics we surface uses the shared +# tools.ansi_strip.strip_ansi (full ECMA-48: CSI, OSC, DCS/SOS/PM/APC, +# C1) so a control sequence can't reposition the cursor or hide text +# after a redaction marker. # Env vars the `op` child actually needs. We build a minimal allowlisted env # rather than copying all of os.environ (which, post-dotenv, holds every @@ -183,16 +183,16 @@ def _auth_fingerprint(token_env: str) -> str: previous identity is never served under a new one. Never logged or displayed; the raw token never leaves this hash. """ - environ = source_environ() + source_env = get_source_environment() parts: List[str] = [ - f"token={environ.get(token_env, '')}", - f"account={environ.get('OP_ACCOUNT', '')}", - f"connect_host={environ.get('OP_CONNECT_HOST', '')}", - f"connect_token={environ.get('OP_CONNECT_TOKEN', '')}", + f"token={source_env.get(token_env, '')}", + f"account={source_env.get('OP_ACCOUNT', '')}", + f"connect_host={source_env.get('OP_CONNECT_HOST', '')}", + f"connect_token={source_env.get('OP_CONNECT_TOKEN', '')}", ] - for key in sorted(environ): + for key in sorted(source_env): if key.startswith("OP_SESSION_"): - parts.append(f"{key}={environ[key]}") + parts.append(f"{key}={source_env[key]}") material = "\n".join(parts) return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16] @@ -232,12 +232,15 @@ def find_op(binary_path: str = "") -> Optional[Path]: def _scrub(text: str) -> str: """Remove ANSI control sequences and trim, for safe message surfacing.""" - return _ANSI_CSI_RE.sub("", text).replace("\x1b", "").strip() + from tools.ansi_strip import strip_ansi + + # strip_ansi removes well-formed sequences; drop any stray lone ESC too. + return strip_ansi(text).replace("\x1b", "").strip() def _op_child_env(token_value: str) -> Dict[str, str]: """Build a minimal allowlisted environment for the ``op`` child process.""" - source_env = source_environ() + source_env = get_source_environment() env: Dict[str, str] = {} for key in _OP_ENV_ALLOWLIST: val = source_env.get(key) @@ -340,7 +343,7 @@ def fetch_onepassword_secrets( if not valid: return {}, warnings - token_value = source_environ().get(token_env, "").strip() + token_value = get_source_environment().get(token_env, "").strip() cache_key: _CacheKey = ( _auth_fingerprint(token_env), account or "", diff --git a/agent/secret_sources/registry.py b/agent/secret_sources/registry.py index 08dbcbfb1c24..216db858baa2 100644 --- a/agent/secret_sources/registry.py +++ b/agent/secret_sources/registry.py @@ -32,7 +32,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, MutableMapping, Optional from agent.secret_sources.base import ( SECRET_SOURCE_API_VERSION, @@ -40,7 +40,8 @@ FetchResult, SecretSource, is_valid_env_name, - use_source_environ, + reset_source_environment, + set_source_environment, ) logger = logging.getLogger(__name__) @@ -197,10 +198,8 @@ def _reset_registry_for_tests() -> None: def _fetch_with_timeout( - source: SecretSource, - cfg: dict, - home_path: Path, - environ: Optional[Dict[str, str]] = None, + source: SecretSource, cfg: dict, home_path: Path, + environ: MutableMapping[str, str], ) -> FetchResult: """Run source.fetch() under a wall-clock budget; never raises. @@ -214,14 +213,14 @@ def _fetch_with_timeout( executor = concurrent.futures.ThreadPoolExecutor( max_workers=1, thread_name_prefix=f"secret-src-{source.name}" ) - - def _fetch(): - if environ is None: - return source.fetch(cfg, home_path) - with use_source_environ(environ): - return source.fetch(cfg, home_path) - try: + def _fetch() -> FetchResult: + token = set_source_environment(environ) + try: + return source.fetch(cfg, home_path) + finally: + reset_source_environment(token) + future = executor.submit(_fetch) try: result = future.result(timeout=timeout) @@ -332,7 +331,7 @@ def _profile_alias_target(var: str, profile: str) -> Optional[str]: def apply_all(secrets_cfg: dict, home_path: Path, - environ: Optional[Dict[str, str]] = None) -> ApplyReport: + environ: Optional[MutableMapping[str, str]] = None) -> ApplyReport: """Fetch from every enabled source and apply the merged result to env. ``environ`` defaults to ``os.environ``; injectable for tests. diff --git a/agent/session_activity.py b/agent/session_activity.py new file mode 100644 index 000000000000..243f30a5a499 --- /dev/null +++ b/agent/session_activity.py @@ -0,0 +1,106 @@ +"""Shared session activity observation contract (#72016 / #72039). + +Observation-only: timestamp + bounded description/provenance. +Notification, timeout, kill, and retry policy stay in their own components. +Consumers distinguish work (API / tool / compacting / stalled) from the +description text itself — there is no separate phase enum. + +Provenance is a small closed enum of *noun* sources (where the stamp came +from). The default agent activity clock (``_touch_activity``) stamps +``unknown`` unless a caller passes an explicit ``provenance=``; named +values are for special writers. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any, Mapping, Optional + +ACTIVITY_DESCRIPTION_MAX = 120 + +# Durable SessionDB activity heartbeat cadence (seconds between writes per +# session). Contract: MUST stay >= 30s — the SessionDB write path is +# contended (deadline/patience retry, compression-lock patience), and the +# heartbeat is an observation-only projection that never justifies extra +# write pressure. This cadence is deliberately a code constant, independent +# of any compression.* or agent.* config, so no configuration can turn the +# heartbeat into a high-frequency writer. Matches the kanban auto-heartbeat +# cadence. force_persist (terminal stamps) is the only bypass. +SESSION_ACTIVITY_HEARTBEAT_MIN_INTERVAL_SECONDS = 60.0 + + +class ActivityProvenance(str, Enum): + """Where a durable/in-memory activity stamp came from.""" + + UNKNOWN = "unknown" + # Compression writers (#72424 / activity contract): heartbeat, host timeout, cooldown. + AGENT_COMPRESSION = "agent.compression" + AGENT_COMPRESSION_TIMEOUT = "agent.compression_timeout" + AGENT_COMPRESSION_COOLDOWN = "agent.compression_cooldown" + + +def bound_activity_description(description: Optional[str]) -> str: + """Clamp free-form activity text to the shared description budget.""" + text = (description or "").strip() + if len(text) <= ACTIVITY_DESCRIPTION_MAX: + return text + return text[: ACTIVITY_DESCRIPTION_MAX - 1] + "…" + + +def normalize_activity_provenance( + provenance: Optional[ActivityProvenance | str], +) -> ActivityProvenance: + """Return a known provenance, or ``UNKNOWN`` when unset/unrecognized.""" + if isinstance(provenance, ActivityProvenance): + return provenance + value = (provenance or "").strip() + try: + return ActivityProvenance(value) + except ValueError: + return ActivityProvenance.UNKNOWN + + +def reset_session_activity_persist_window(agent: Any) -> None: + """Clear the agent's durable SessionDB activity persist rate-limit window. + + The next ``_touch_activity`` / ``_persist_session_activity_if_due`` will + write through even if a stamp landed within the last 60s. Used for + terminal compression labels that must not stay stuck on mid-compress + text (e.g. "context compression in progress" after /compress). + """ + try: + agent._session_activity_last_persist_mono = 0.0 + except Exception: + pass + + +def build_activity_snapshot( + *, + last_activity_at: Optional[float], + last_activity_description: Optional[str], + last_activity_provenance: Optional[ActivityProvenance | str] = None, + now: Optional[float] = None, + extra: Optional[Mapping[str, Any]] = None, +) -> dict[str, Any]: + """Build the shared activity snapshot (plus optional caller extras).""" + import time as _time + + when = float(last_activity_at) if last_activity_at is not None else None + clock = float(now if now is not None else _time.time()) + desc = bound_activity_description(last_activity_description) + prov = normalize_activity_provenance(last_activity_provenance) + elapsed = round(clock - when, 1) if when is not None else None + snap: dict[str, Any] = { + "last_activity_at": when, + "last_activity_description": desc, + "last_activity_provenance": prov.value, + "seconds_since_activity": elapsed, + # Short aliases used by existing gateway/delegate readers. + "last_activity_ts": when, + "last_activity_desc": desc, + "description": desc, + "provenance": prov.value, + } + if extra: + snap.update(dict(extra)) + return snap diff --git a/agent/shell_hooks.py b/agent/shell_hooks.py index c1cec81df333..db44ddab2b90 100644 --- a/agent/shell_hooks.py +++ b/agent/shell_hooks.py @@ -100,6 +100,7 @@ child_role – role string of the child agent child_summary – summary of the child's work child_status – exit status string (e.g. "success", "error") + tool_call_history – redacted tool name/input summary/byte counts/status list duration_ms – wall-clock time of the child run in milliseconds """ @@ -318,8 +319,9 @@ def _parse_hooks_block(hooks_cfg: Any) -> List[ShellHookSpec]: for event_name, entries in hooks_cfg.items(): # Reserved sub-keys that aren't event names — skip silently. These # are config sub-sections nested under `hooks:` for related - # functionality (e.g. output-spill budgets). - if event_name in ("output_spill",): + # functionality (e.g. output-spill budgets, outbound webhooks — + # the latter parsed by agent/outbound_webhooks.py). + if event_name in ("output_spill", "outbound"): continue if event_name not in VALID_HOOKS: suggestion = difflib.get_close_matches( @@ -464,7 +466,7 @@ def _spawn(spec: ShellHookSpec, stdin_json: str) -> Dict[str, Any]: input=stdin_json, capture_output=True, timeout=spec.timeout, - text=True, + text=True, encoding='utf-8', errors='replace', shell=False, **_popen_kwargs, ) @@ -632,7 +634,7 @@ def allowlist_path() -> Path: def load_allowlist() -> Dict[str, Any]: """Return the parsed allowlist, or an empty skeleton if absent.""" try: - raw = json.loads(allowlist_path().read_text()) + raw = json.loads(allowlist_path().read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError, OSError): return {"approvals": []} if not isinstance(raw, dict): diff --git a/agent/skill_commands.py b/agent/skill_commands.py index fa1b4044a7ae..3f1156a85929 100644 --- a/agent/skill_commands.py +++ b/agent/skill_commands.py @@ -54,6 +54,21 @@ _BUNDLE_USER_INSTRUCTION = "\nUser instruction: " _BUNDLE_FIRST_SKILL_BLOCK = "\n\n[Loaded as part of the " +# The skill name sits in the first quoted span of the activation note, for both +# the single-skill and the bundle header ("work" / "/clean /work"). +_SKILL_NAME_RE = re.compile(re.escape(_SKILL_INVOCATION_PREFIX) + r'"([^"]*)"') + +# SQL LIKE pattern matching a skill-expanded turn, for listing queries that +# have to recognize scaffolding before the row reaches Python. The prefix +# contains no LIKE wildcards (`%`, `_`), so it needs no ESCAPE clause. +SKILL_SCAFFOLD_SQL_LIKE = _SKILL_INVOCATION_PREFIX + "%" + +# Marks where a preview query joined the head and tail of a long scaffolded +# message. ``describe_skill_invocation`` may hand back a span that runs across +# the joint (a bundle instruction cut off by the head window); callers cut the +# description there rather than show the skill body on the far side. +SKILL_EXCERPT_JOINT = "\x1e" + def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]: """Recover the user's instruction from a slash-skill-expanded turn. @@ -82,6 +97,45 @@ def extract_user_instruction_from_skill_message(content: Any) -> Optional[str]: return None +def describe_skill_invocation(content: Any, separator: str = " — ") -> Optional[str]: + """Render a slash-skill-expanded turn the way the user typed it. + + The expanded message embeds the whole skill body, so any surface that + summarizes a user turn from its raw content — session titles, sidebar + previews, the ``/rewind`` picker — otherwise shows the skill's own prose + as if the user had written it. That is how a skill's opening line ends up + as a session title. + + Returns ``"/work — fix the title leak"``, or ``"/work"`` for a bare + invocation, or ``None`` when *content* is not skill scaffolding (the + caller should then summarize it as an ordinary message). + + *separator* joins the command and the instruction. Previews use the + default em dash; pass ``" "`` for the literal invocation the user typed, + which is what chat transcripts render. + """ + if not isinstance(content, str) or not content.startswith(_SKILL_INVOCATION_PREFIX): + return None + + match = _SKILL_NAME_RE.match(content) + name = (match.group(1) if match else "").strip() + # Bundle headers already carry their typed "/a /b" keys; a single skill is + # a bare name. + label = name if name.startswith("/") else f"/{name}" + + instruction = extract_user_instruction_from_skill_message(content) + if instruction and instruction is not content: + # An excerpted message (head + tail, joined by SKILL_EXCERPT_JOINT) can + # put the joint inside the matched span — keep only the side the + # instruction marker was found on. + instruction = instruction.split(SKILL_EXCERPT_JOINT)[0] + instruction = " ".join(instruction.split()) + if instruction: + return f"{label}{separator}{instruction}" if name else instruction + + return label if name else None + + def _extract_single_skill_user_instruction(message: str) -> Optional[str]: # Single-skill format appends the user instruction after the skill body, so # the last occurrence is the user-provided one; the body may quote this text. diff --git a/agent/skill_preprocessing.py b/agent/skill_preprocessing.py index bd0386d58058..44c5714b9b39 100644 --- a/agent/skill_preprocessing.py +++ b/agent/skill_preprocessing.py @@ -25,9 +25,9 @@ def load_skills_config() -> dict: """Load the ``skills`` section of config.yaml (best-effort).""" try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - cfg = load_config() or {} + cfg = load_config_readonly() or {} skills_cfg = cfg.get("skills") if isinstance(skills_cfg, dict): return skills_cfg @@ -74,7 +74,7 @@ def run_inline_shell(command: str, cwd: Path | None, timeout: int) -> str: ["bash", "-c", command], cwd=str(cwd) if cwd else None, capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=max(1, int(timeout)), check=False, stdin=subprocess.DEVNULL, diff --git a/agent/skill_utils.py b/agent/skill_utils.py index df0f933317fc..a302c6981a47 100644 --- a/agent/skill_utils.py +++ b/agent/skill_utils.py @@ -49,8 +49,57 @@ # archive workflow preserves a complete old skill package under references/. SKILL_SUPPORT_DIRS = frozenset(("references", "templates", "assets", "scripts")) +# ── Org-shared skills (sync contract) ─────────────────────────── +# Org mirrors live under ~/.hermes/skills/_org//. Resolution is +# TOKEN-GATED via a marker file the sync client writes after verifying the +# token (skills_sync_client.pull_org_skills): only the marked org's mirror is +# scanned. No marker ⇒ no org skills load. The marker is plain data (org_id +# string) so this module stays import-light; the VERIFICATION lives in the +# sync client, which is the only writer. Offline grace: the marker persists, +# so already-pulled org skills keep working without connectivity; a VERIFIED +# org change (or personal-org token) rewrites/removes it. + +ORG_MIRROR_DIR_NAME = "_org" +ORG_ACTIVE_MARKER = ".active_org" +ORG_PROVENANCE_FILE = ".org-provenance.json" +# Records the fingerprint of each skill exactly as upstream sent it, so a +# later local edit is detectable and an org pull can refuse to clobber it. +ORG_BASELINE_FILE = ".org-baseline.json" + + +def read_active_org_id(skills_dir: Path) -> Optional[str]: + """The org id whose mirror may resolve, or None (no org skills load).""" + try: + marker = skills_dir / ORG_MIRROR_DIR_NAME / ORG_ACTIVE_MARKER + if not marker.exists(): + return None + val = marker.read_text(encoding="utf-8").strip() + return val or None + except OSError: + return None + + +def is_org_mirror_path(path, skills_dir: Path) -> bool: + """True when *path* is inside the org mirror (``_org/``).""" + try: + rel = Path(path).resolve().relative_to(Path(skills_dir).resolve()) + except (OSError, ValueError): + return False + return bool(rel.parts) and rel.parts[0] == ORG_MIRROR_DIR_NAME + + +def org_id_of_path(path, skills_dir: Path) -> Optional[str]: + """The ```` segment for a path under ``_org//...``.""" + try: + rel = Path(path).resolve().relative_to(Path(skills_dir).resolve()) + except (OSError, ValueError): + return None + if len(rel.parts) >= 2 and rel.parts[0] == ORG_MIRROR_DIR_NAME: + return rel.parts[1] + return None + -def is_excluded_skill_path(path) -> bool: +def is_excluded_skill_path(path, *, root: Optional[Path] = None) -> bool: """True if *path* should be skipped by active skill scanners. Use this on every ``SKILL.md`` path produced by direct ``rglob`` scans to @@ -66,11 +115,11 @@ def is_excluded_skill_path(path) -> bool: from pathlib import PurePath parts = PurePath(str(path)).parts return any(part in EXCLUDED_SKILL_DIRS for part in parts) or is_skill_support_path( - path + path, root=root ) -def is_skill_support_path(path) -> bool: +def is_skill_support_path(path, *, root: Optional[Path] = None) -> bool: """True if *path* is under a support dir of an actual skill root. ``references/``, ``templates/``, ``assets/``, and ``scripts/`` are @@ -92,6 +141,8 @@ def is_skill_support_path(path) -> bool: if part not in SKILL_SUPPORT_DIRS or idx == 0: continue skill_root = Path(*parts[:idx]) + if root is not None and not path_obj.is_absolute(): + skill_root = root / skill_root if (skill_root / "SKILL.md").exists(): return True return False @@ -815,11 +866,24 @@ def iter_skill_index_files(skills_dir: Path, filename: str): scripts) can contain arbitrary markdown and even archived package ``SKILL.md`` files, but they are progressive-disclosure data loaded through ``skill_view(..., file_path=...)`` rather than active skill roots. + + M2 org mirrors (``_org/``): TOKEN-GATED resolution. Only the active org's + subdir (per the sync-client-written ``.active_org`` marker) is walked; + every other ``_org//`` (stale mirror from a previous org, or no + marker at all) is pruned — leave an org and its skills stop resolving, + without any manual cleanup. """ skills_dir_str = str(skills_dir) + active_org = read_active_org_id(skills_dir) + org_root = os.path.join(skills_dir_str, ORG_MIRROR_DIR_NAME) matches: list[str] = [] for root, dirs, files in os.walk(skills_dir_str, followlinks=True): has_skill_md = "SKILL.md" in files + if root == skills_dir_str and ORG_MIRROR_DIR_NAME in dirs and active_org is None: + dirs.remove(ORG_MIRROR_DIR_NAME) + elif root == org_root: + # Inside _org/: descend ONLY into the active org's mirror. + dirs[:] = [d for d in dirs if d == active_org] dirs[:] = [ d for d in dirs diff --git a/agent/ssl_guard.py b/agent/ssl_guard.py index 557f8566c32a..ac1b7841b2d8 100644 --- a/agent/ssl_guard.py +++ b/agent/ssl_guard.py @@ -31,7 +31,8 @@ def _skip_ssl_guard_enabled() -> bool: def _repair_hint() -> str: return ( - "Repair: python -m pip install --force-reinstall certifi openai httpx\n" + "Repair: run `hermes doctor --fix` (auto-reinstalls certifi), or " + "manually: python -m pip install --force-reinstall certifi openai httpx\n" "If you configured a custom corporate CA bundle, fix or unset the " "broken CA bundle environment variable." ) diff --git a/agent/subagent_lifecycle.py b/agent/subagent_lifecycle.py new file mode 100644 index 000000000000..55e110aae5a6 --- /dev/null +++ b/agent/subagent_lifecycle.py @@ -0,0 +1,540 @@ +"""Public, plugin-safe lifecycle API for delegated Hermes subagents. + +This module deliberately exposes immutable contracts, not ``AIAgent`` objects. +It is the supported boundary for plugins that need to supervise fresh child +sessions; plugins must obtain it from ``PluginContext.subagent_lifecycle``. +""" + +from __future__ import annotations + +import contextvars +import dataclasses +import enum +import hashlib +import hmac +import json +import math +import secrets +import threading +import time +from contextlib import contextmanager +from concurrent.futures import Future, TimeoutError +from typing import Any, Callable, Mapping, Optional + +from agent.interrupt_compat import request_hard_interrupt + +PUBLIC_CONTRACT_VERSION = 1 +_MAX_GOAL_CHARS = 16_000 +_MAX_CONTEXT_CHARS = 32_000 +_MAX_METADATA_BYTES = 8_192 +_MAX_RESULT_CHARS = 32_000 +_TERMINAL_RETENTION_SECONDS = 3_600 + + +class SubagentLifecycleError(ValueError): + """A request cannot be safely accepted by the public lifecycle API.""" + + +class SubagentState(str, enum.Enum): + PENDING = "PENDING" + STARTING = "STARTING" + RUNNING = "RUNNING" + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" + INTERRUPTED = "INTERRUPTED" + CANCEL_REQUESTED = "CANCEL_REQUESTED" + CANCELLED = "CANCELLED" + UNKNOWN = "UNKNOWN" + + +@dataclasses.dataclass(frozen=True) +class SubagentLaunchRequest: + goal: str + context: Optional[str] = None + role: str = "leaf" + model: Optional[str] = None + allowed_toolsets: Optional[tuple[str, ...]] = None + blocked_tools: tuple[str, ...] = () + working_directory: Optional[str] = None + parent_session_id: Optional[str] = None + correlation_id: Optional[str] = None + metadata: Mapping[str, Any] = dataclasses.field(default_factory=dict) + timeout_seconds: Optional[float] = None + + +@dataclasses.dataclass(frozen=True) +class SubagentHandle: + contract_version: int + subagent_id: str + parent_session_id: Optional[str] + correlation_id: Optional[str] + created_at: float + provider: Optional[str] + model: Optional[str] + role: str + depth: int + capability: str + + def to_dict(self) -> dict[str, Any]: + return dataclasses.asdict(self) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "SubagentHandle": + try: + return cls(**dict(value)) + except (TypeError, ValueError) as exc: + raise SubagentLifecycleError("Malformed subagent handle.") from exc + + +@dataclasses.dataclass(frozen=True) +class SubagentStatus: + handle: SubagentHandle + state: SubagentState + updated_at: float + diagnostic: Optional[str] = None + + +@dataclasses.dataclass(frozen=True) +class SubagentTerminalState: + handle: SubagentHandle + state: SubagentState + completed: bool + timed_out: bool = False + diagnostic: Optional[str] = None + + +@dataclasses.dataclass(frozen=True) +class SubagentCancelResult: + accepted: bool + already_terminal: bool = False + unknown_handle: bool = False + unsupported: bool = False + state: SubagentState = SubagentState.UNKNOWN + + +@dataclasses.dataclass(frozen=True) +class SubagentResult: + handle: SubagentHandle + terminal_state: SubagentState + ready: bool + summary: Optional[str] = None + structured_payload: Optional[Mapping[str, Any]] = None + started_at: Optional[float] = None + completed_at: Optional[float] = None + error_classification: Optional[str] = None + error_message: Optional[str] = None + usage_metadata: Mapping[str, Any] = dataclasses.field(default_factory=dict) + tool_execution_summary: Mapping[str, Any] = dataclasses.field(default_factory=dict) + result_hash: Optional[str] = None + + +@dataclasses.dataclass(frozen=True) +class SubagentReconnectResult: + connected: bool + state: SubagentState + diagnostic: Optional[str] = None + + +@dataclasses.dataclass +class _Record: + handle: SubagentHandle + state: SubagentState + updated_at: float + agent: Any = None + future: Optional[Future] = None + started_at: Optional[float] = None + completed_at: Optional[float] = None + result: Optional[SubagentResult] = None + + +class _Registry: + """Thread-safe terminal-retention registry; never returns live records.""" + + def __init__(self) -> None: + self.lock = threading.RLock() + self.records: dict[str, _Record] = {} + self.correlations: dict[tuple[Optional[str], str], str] = {} + + +_REGISTRY = _Registry() +# Daemon worker pool: a wedged/abandoned child must never block interpreter +# exit at atexit-join time (same rationale as _run_single_child's timeout +# executor and the async-delegation registry pool). +from tools.daemon_pool import DaemonThreadPoolExecutor as _DaemonExecutor + +_EXECUTOR = _DaemonExecutor(max_workers=8, thread_name_prefix="hermes-lifecycle") +_SECRET = secrets.token_bytes(32) +_ACTIVE_PARENT_AGENT: contextvars.ContextVar[Any] = contextvars.ContextVar( + "hermes_subagent_lifecycle_parent", default=None +) + + +@contextmanager +def bind_subagent_parent(parent_agent: Any): + """Bind the host-owned parent for the current agent turn.""" + token = _ACTIVE_PARENT_AGENT.set(parent_agent) + try: + yield + finally: + _ACTIVE_PARENT_AGENT.reset(token) + + +def get_active_subagent_parent() -> Any: + """Return the parent bound to this execution context, if any.""" + return _ACTIVE_PARENT_AGENT.get() + + +class SubagentLifecycleService: + """Stable public service returned by :attr:`PluginContext.subagent_lifecycle`. + + Running children are in-process only. Completed results remain available + until process exit; ``reconnect`` accurately reports that a serialized + handle cannot reconnect after a restart instead of launching work again. + """ + + def __init__(self, parent_agent_resolver: Callable[[], Any]) -> None: + self._parent_agent_resolver = parent_agent_resolver + + def launch(self, request: SubagentLaunchRequest) -> SubagentHandle: + parent = self._parent_agent_resolver() + if parent is None: + raise SubagentLifecycleError( + "No active Hermes parent session is available." + ) + self._validate_request(request, parent) + parent_session_id = str(getattr(parent, "session_id", "") or "") or None + if request.parent_session_id and request.parent_session_id != parent_session_id: + raise SubagentLifecycleError( + "parent_session_id does not match the active session." + ) + correlation_key = (parent_session_id, request.correlation_id or "") + with _REGISTRY.lock: + self._cleanup_locked() + if request.correlation_id and correlation_key in _REGISTRY.correlations: + raise SubagentLifecycleError( + "Duplicate correlation_id for this parent session." + ) + + # Delegate construction remains internal so plugin code never imports + # private delegation helpers or manipulates the active-child registry. + from tools.delegate_tool import ( + _build_child_preserving_parent_tools, + DEFAULT_MAX_ITERATIONS, + ) + + child = _build_child_preserving_parent_tools( + task_index=0, + goal=request.goal, + context=request.context, + toolsets=list(request.allowed_toolsets) + if request.allowed_toolsets + else None, + model=request.model, + max_iterations=DEFAULT_MAX_ITERATIONS, + task_count=1, + parent_agent=parent, + role=request.role, + ) + subagent_id = str(getattr(child, "_subagent_id", "") or "") + if not subagent_id: + raise SubagentLifecycleError("Hermes failed to assign a child identity.") + created = time.time() + handle = SubagentHandle( + PUBLIC_CONTRACT_VERSION, + subagent_id, + parent_session_id, + request.correlation_id, + created, + getattr(child, "provider", None), + getattr(child, "model", None), + getattr(child, "_delegate_role", request.role), + int(getattr(child, "_delegate_depth", 1) or 1), + self._capability(subagent_id, parent_session_id, created), + ) + record = _Record(handle, SubagentState.PENDING, created, agent=child) + with _REGISTRY.lock: + _REGISTRY.records[subagent_id] = record + if request.correlation_id: + _REGISTRY.correlations[correlation_key] = subagent_id + record.future = _EXECUTOR.submit(self._run, record, request.goal, parent) + return handle + + def status(self, handle: SubagentHandle) -> SubagentStatus: + record = self._record(handle) + if record is None: + return SubagentStatus( + handle, SubagentState.UNKNOWN, time.time(), "UNKNOWN_HANDLE" + ) + with _REGISTRY.lock: + return SubagentStatus(record.handle, record.state, record.updated_at) + + def wait( + self, handle: SubagentHandle, *, timeout_seconds: Optional[float] = None + ) -> SubagentTerminalState: + record = self._record(handle) + if record is None: + return SubagentTerminalState( + handle, SubagentState.UNKNOWN, True, diagnostic="UNKNOWN_HANDLE" + ) + future = record.future + if future is not None: + try: + future.result(timeout=timeout_seconds) + except TimeoutError: + return SubagentTerminalState(record.handle, record.state, False, True) + except Exception: + pass + with _REGISTRY.lock: + return SubagentTerminalState( + record.handle, record.state, record.result is not None + ) + + def cancel(self, handle: SubagentHandle, *, reason: str) -> SubagentCancelResult: + record = self._record(handle) + if record is None: + return SubagentCancelResult(False, unknown_handle=True) + with _REGISTRY.lock: + if record.result is not None: + return SubagentCancelResult( + False, already_terminal=True, state=record.state + ) + agent = record.agent + record.state = SubagentState.CANCEL_REQUESTED + record.updated_at = time.time() + if agent is None: + return SubagentCancelResult( + False, unsupported=True, state=SubagentState.CANCEL_REQUESTED + ) + try: + accepted = request_hard_interrupt( + agent, f"Lifecycle cancellation requested: {reason[:500]}" + ) + except Exception: + return SubagentCancelResult( + False, unsupported=True, state=SubagentState.CANCEL_REQUESTED + ) + if not accepted: + return SubagentCancelResult( + False, unsupported=True, state=SubagentState.CANCEL_REQUESTED + ) + return SubagentCancelResult(True, state=SubagentState.CANCEL_REQUESTED) + + def result(self, handle: SubagentHandle) -> SubagentResult: + record = self._record(handle) + if record is None: + return SubagentResult( + handle, + SubagentState.UNKNOWN, + False, + error_classification="UNKNOWN_HANDLE", + ) + with _REGISTRY.lock: + if record.result is not None: + return record.result + return SubagentResult( + record.handle, record.state, False, error_classification="NOT_READY" + ) + + def reconnect(self, handle: SubagentHandle) -> SubagentReconnectResult: + record = self._record(handle) + if record is None: + return SubagentReconnectResult( + False, SubagentState.UNKNOWN, "RECONNECT_UNAVAILABLE" + ) + with _REGISTRY.lock: + return SubagentReconnectResult(True, record.state) + + def _record(self, handle: SubagentHandle) -> Optional[_Record]: + if ( + not isinstance(handle, SubagentHandle) + or type(handle.contract_version) is not int + or handle.contract_version != PUBLIC_CONTRACT_VERSION + ): + return None + if ( + not isinstance(handle.subagent_id, str) + or not handle.subagent_id + or ( + handle.parent_session_id is not None + and not isinstance(handle.parent_session_id, str) + ) + or ( + handle.correlation_id is not None + and not isinstance(handle.correlation_id, str) + ) + or isinstance(handle.created_at, bool) + or not isinstance(handle.created_at, (int, float)) + or not math.isfinite(handle.created_at) + or (handle.provider is not None and not isinstance(handle.provider, str)) + or (handle.model is not None and not isinstance(handle.model, str)) + or not isinstance(handle.role, str) + or type(handle.depth) is not int + or not isinstance(handle.capability, str) + ): + return None + if not hmac.compare_digest( + handle.capability, + self._capability( + handle.subagent_id, handle.parent_session_id, handle.created_at + ), + ): + return None + parent = self._parent_agent_resolver() + active_parent_id = str(getattr(parent, "session_id", "") or "") or None + if active_parent_id != handle.parent_session_id: + return None + with _REGISTRY.lock: + return _REGISTRY.records.get(handle.subagent_id) + + @staticmethod + def _cleanup_locked() -> None: + """Retain terminal snapshots for a bounded period, never live work.""" + cutoff = time.time() - _TERMINAL_RETENTION_SECONDS + expired = [ + subagent_id + for subagent_id, record in _REGISTRY.records.items() + if record.result is not None + and record.completed_at is not None + and record.completed_at < cutoff + ] + for subagent_id in expired: + record = _REGISTRY.records.pop(subagent_id) + if record.handle.correlation_id: + _REGISTRY.correlations.pop( + (record.handle.parent_session_id, record.handle.correlation_id), + None, + ) + + def _run(self, record: _Record, goal: str, parent: Any) -> None: + with _REGISTRY.lock: + if record.state is not SubagentState.CANCEL_REQUESTED: + record.state = SubagentState.RUNNING + record.started_at = time.time() + record.updated_at = record.started_at + try: + from tools.delegate_tool import _run_child_lifecycle + + raw = _run_child_lifecycle(0, goal, record.agent, parent) + status = ( + str(raw.get("status", "error")) if isinstance(raw, dict) else "error" + ) + if status == "completed": + state = SubagentState.SUCCEEDED + elif status == "interrupted": + state = ( + SubagentState.CANCELLED + if record.state == SubagentState.CANCEL_REQUESTED + else SubagentState.INTERRUPTED + ) + else: + state = SubagentState.FAILED + summary = raw.get("summary") if isinstance(raw, dict) else None + summary = str(summary)[:_MAX_RESULT_CHARS] if summary is not None else None + error = raw.get("error") if isinstance(raw, dict) else None + result = SubagentResult( + record.handle, + state, + True, + summary=summary, + completed_at=time.time(), + started_at=record.started_at, + error_classification=None + if state == SubagentState.SUCCEEDED + else status.upper(), + error_message=str(error)[:_MAX_RESULT_CHARS] if error else None, + usage_metadata={"api_calls": raw.get("api_calls", 0)} + if isinstance(raw, dict) + else {}, + tool_execution_summary={ + "duration_seconds": raw.get("duration_seconds", 0) + } + if isinstance(raw, dict) + else {}, + ) + except Exception as exc: + result = SubagentResult( + record.handle, + SubagentState.FAILED, + True, + started_at=record.started_at, + completed_at=time.time(), + error_classification=type(exc).__name__, + error_message=str(exc)[:_MAX_RESULT_CHARS], + ) + payload = dataclasses.asdict(result) + payload.pop("result_hash", None) + result = dataclasses.replace( + result, + result_hash=hashlib.sha256( + json.dumps(payload, sort_keys=True, default=str).encode() + ).hexdigest(), + ) + with _REGISTRY.lock: + record.agent = None + record.result = result + record.state = result.terminal_state + record.completed_at = result.completed_at + record.updated_at = result.completed_at or time.time() + + @staticmethod + def _capability( + subagent_id: str, parent_session_id: Optional[str], created_at: float + ) -> str: + value = f"{subagent_id}|{parent_session_id or ''}|{created_at:.6f}".encode() + return hmac.new(_SECRET, value, hashlib.sha256).hexdigest() + + @staticmethod + def _validate_request(request: SubagentLaunchRequest, parent: Any) -> None: + if ( + not isinstance(request, SubagentLaunchRequest) + or not isinstance(request.goal, str) + or not request.goal.strip() + or len(request.goal) > _MAX_GOAL_CHARS + ): + raise SubagentLifecycleError( + "goal must be a non-empty string of at most 16000 characters." + ) + if request.context is not None and ( + not isinstance(request.context, str) + or len(request.context) > _MAX_CONTEXT_CHARS + ): + raise SubagentLifecycleError( + "context must be a string of at most 32000 characters." + ) + if request.role not in {"leaf", "orchestrator"}: + raise SubagentLifecycleError("role must be 'leaf' or 'orchestrator'.") + if request.timeout_seconds is not None: + raise SubagentLifecycleError( + "Per-launch timeout is not supported; configure delegation timeout explicitly." + ) + if request.working_directory is not None: + raise SubagentLifecycleError( + "working_directory is not supported because Hermes delegates use isolated task environments." + ) + if request.blocked_tools: + raise SubagentLifecycleError( + "Per-tool blocking is not supported; use allowed_toolsets. Hermes always blocks unsafe child tools." + ) + try: + metadata_bytes = len( + json.dumps(dict(request.metadata), sort_keys=True).encode() + ) + except (TypeError, ValueError) as exc: + raise SubagentLifecycleError("metadata must be JSON-serializable.") from exc + if metadata_bytes > _MAX_METADATA_BYTES: + raise SubagentLifecycleError("metadata exceeds 8192 bytes.") + if request.allowed_toolsets: + from toolsets import TOOLSETS + + unknown = set(request.allowed_toolsets) - set(TOOLSETS) + if unknown: + raise SubagentLifecycleError( + f"Unknown toolsets: {', '.join(sorted(unknown))}." + ) + enabled = getattr(parent, "enabled_toolsets", None) + if enabled is not None and not set(request.allowed_toolsets).issubset( + set(enabled) + ): + raise SubagentLifecycleError( + "Requested toolsets would broaden parent permissions." + ) diff --git a/agent/subdirectory_hints.py b/agent/subdirectory_hints.py index ca96c664cb51..4e9f7f5ed335 100644 --- a/agent/subdirectory_hints.py +++ b/agent/subdirectory_hints.py @@ -13,6 +13,7 @@ Inspired by Block/goose's SubdirectoryHintTracker. """ +import hashlib import logging import os import shlex @@ -45,6 +46,18 @@ # Prevents scanning all the way to / for deeply nested paths. _MAX_ANCESTOR_WALK = 5 +# Directory names that never contain authoritative project context. +# Backups, vendored deps, VCS internals, and caches routinely hold *copies* of +# AGENTS.md; loading those duplicates real context and inflates the prompt. +_EXCLUDED_DIR_NAMES = frozenset({ + "node_modules", "venv", ".venv", "__pycache__", + ".git", ".hg", ".svn", + ".Trash", ".cache", ".tox", ".mypy_cache", ".pytest_cache", + "site-packages", "dist-packages", + "backups", "backup", ".backups", + "vendor", "third_party", +}) + def _is_ancestor_or_same(a: Path, b: Path) -> bool: """Check if *a* is the same as or an ancestor of *b* (parent directory check).""" @@ -54,6 +67,7 @@ def _is_ancestor_or_same(a: Path, b: Path) -> bool: except ValueError: return False + class SubdirectoryHintTracker: """Track which directories the agent visits and load hints on first access. @@ -70,8 +84,34 @@ class SubdirectoryHintTracker: def __init__(self, working_dir: Optional[str] = None): self.working_dir = Path(working_dir or os.getcwd()).resolve() self._loaded_dirs: Set[Path] = set() + # Content digests already injected — prevents re-sending the same file + # reachable through symlinks, hardlinks, or duplicated copies. + self._loaded_digests: Set[str] = set() # Pre-mark the working dir as loaded (startup context handles it) self._loaded_dirs.add(self.working_dir) + self._seed_working_dir_digest() + + def _seed_working_dir_digest(self) -> None: + """Record the CWD context file's digest so it is never re-injected. + + ``prompt_builder`` already loads the working directory's context file at + startup. Seeding its digest here means the same content reached through + a different path (a symlink farm, a shared workspace) is recognised as a + duplicate instead of being sent a second time. + """ + for filename in _HINT_FILENAMES: + candidate = self.working_dir / filename + try: + if not candidate.is_file(): + continue + content = candidate.read_text(encoding="utf-8").strip() + except (OSError, UnicodeDecodeError): + continue + if content: + self._loaded_digests.add( + hashlib.sha256(content.encode("utf-8")).hexdigest() + ) + break # first match wins, mirroring startup loading def check_tool_call( self, @@ -193,8 +233,25 @@ def _is_valid_subdir(self, path: Path) -> bool: # check as a best-effort safeguard. if not _is_ancestor_or_same(self.working_dir, path): return False + if self._is_excluded(path): + return False return True + def _is_excluded(self, path: Path) -> bool: + """True when the path sits inside a directory that holds copies, not context. + + Directories the user is deliberately working inside are never excluded — + if ``working_dir`` is itself under ``vendor/``, that segment is legitimate + and only segments *below* the working dir are screened. + """ + try: + rel_parts = path.relative_to(self.working_dir).parts + except ValueError: + # Paths outside the working dir are already rejected by + # _is_valid_subdir before this runs; treat as excluded defensively. + return True + return any(part in _EXCLUDED_DIR_NAMES for part in rel_parts) + def _load_hints_for_directory(self, directory: Path) -> Optional[str]: """Load hint files from a directory. Returns formatted text or None. @@ -230,6 +287,19 @@ def _load_hints_for_directory(self, directory: Path) -> Optional[str]: content = hint_path.read_text(encoding="utf-8").strip() if not content: continue + # Skip content we've already injected. The same AGENTS.md is + # routinely reachable through several paths (symlinked shared + # workspaces, hardlinks, copied backups); re-sending it burns + # context for zero new information. + digest = hashlib.sha256(content.encode("utf-8")).hexdigest() + if digest in self._loaded_digests: + logger.debug( + "Skipping duplicate hint content at %s (digest %s)", + hint_path, + digest[:12], + ) + break + self._loaded_digests.add(digest) # Same security scan as startup context loading content = _scan_context_content(content, filename) if len(content) > _MAX_HINT_CHARS: diff --git a/agent/subscription_view.py b/agent/subscription_view.py index a65835983d8c..c2c55e0ee20b 100644 --- a/agent/subscription_view.py +++ b/agent/subscription_view.py @@ -504,3 +504,4 @@ def dev_fixture_subscription_state() -> Optional[SubscriptionState]: # Unknown name → behave as logged-out so the misconfiguration is visible. return SubscriptionState(logged_in=False, error=f"unknown HERMES_DEV_SUBSCRIPTION_FIXTURE: {name}") + diff --git a/agent/system_prompt.py b/agent/system_prompt.py index f1211e426199..35fec9b248a7 100644 --- a/agent/system_prompt.py +++ b/agent/system_prompt.py @@ -12,9 +12,11 @@ * ``stable`` — identity (SOUL.md or DEFAULT_AGENT_IDENTITY), tool guidance, computer-use guidance, nous subscription block, tool-use enforcement guidance + per-model operational guidance, skills prompt, - alibaba model-name workaround, environment hints, platform hints. + alibaba model-name workaround, environment hints, coding guidance, + platform hints. * ``context`` — caller-supplied ``system_message`` plus context files - (AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``. + (AGENTS.md / .cursorrules / etc.) discovered under ``TERMINAL_CWD``, + plus the session's coding-workspace snapshot. * ``volatile`` — memory snapshot, USER.md profile, external memory provider block, timestamp/session/model/provider line. @@ -24,6 +26,7 @@ from __future__ import annotations import json +import logging import os from typing import Any, Dict, List, Optional @@ -49,6 +52,8 @@ from hermes_constants import get_hermes_home from utils import is_truthy_value +logger = logging.getLogger(__name__) + def _ra(): """Lazy reference to the ``run_agent`` module. @@ -145,14 +150,14 @@ def _tui_embedded_pane_clarifier(hint: str) -> str: def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]: - """Assemble the system prompt as three ordered parts. + """Assemble the system prompt as three ordered cache tiers. Returns a dict with three keys: - * ``stable`` — identity, tool guidance, skills prompt, - environment hints, platform hints, model-family operational - guidance. - * ``context`` — context files (AGENTS.md, .cursorrules, etc.) - and caller-supplied system_message. + * ``stable`` — the cross-session-stable prefix, through the coding + operating brief when a workspace snapshot follows. + * ``context`` — the workspace snapshot followed by the remaining + session-stable guidance, context files, and caller-supplied + system_message. * ``volatile`` — memory snapshot, user profile, external memory provider block, timestamp line. @@ -353,25 +358,35 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) stable_parts.append(_env_hints) # Coding posture (base Hermes, any interactive coding surface in a code - # workspace — see agent/coding_context.py). The operating brief + the live - # git/workspace snapshot are built once here and cached for the session; - # the snapshot is never re-probed per turn (that would break the prompt - # cache), so the brief tells the model to re-check git before relying on it. + # workspace — see agent/coding_context.py). Keep the operating brief in + # the cross-session-stable prefix, while placing the live git/workspace + # snapshot behind its own cache boundary. The post-snapshot blocks must + # stay in their historical position after the workspace snapshot. + coding_workspace_parts: List[str] = [] + coding_trailing_parts: List[str] = [] if agent.valid_tool_names: try: - from agent.coding_context import coding_system_blocks - - stable_parts.extend( - coding_system_blocks( - platform=agent.platform, - cwd=resolve_context_cwd(), - model=agent.model, - ) + from agent.coding_context import coding_system_prompt_parts + + coding_prefix_parts, coding_workspace_parts, coding_trailing_parts = coding_system_prompt_parts( + platform=agent.platform, + cwd=resolve_context_cwd(), + model=agent.model, ) + stable_parts.extend(coding_prefix_parts) except Exception: # Coding-context probing must never block prompt build. pass + # Guidance assembled after the coding posture historically followed the + # workspace snapshot. With no snapshot, the coding tail instead remains + # directly after the coding prefix in the cacheable prefix. + if coding_workspace_parts: + post_workspace_parts: List[str] = [] + else: + stable_parts.extend(coding_trailing_parts) + post_workspace_parts = stable_parts + # Local Python toolchain probe — names python/pip/uv/PEP-668 state when # something is non-default so the model can pick the right install # strategy without discovering by failure. Emits a single line; emits @@ -384,7 +399,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) from tools.env_probe import get_environment_probe_line _probe_line = get_environment_probe_line() if _probe_line: - stable_parts.append(_probe_line) + post_workspace_parts.append(_probe_line) except Exception: # Probe failure must never block prompt build. pass @@ -402,7 +417,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) except Exception: active_profile = "default" if active_profile == "default": - stable_parts.append( + post_workspace_parts.append( "Active Hermes profile: default. Other profiles (if any) live " "under " + str(get_hermes_home()) + "/profiles//. Each profile has its own " "skills/, plugins/, cron/, and memories/ that affect a different " @@ -411,7 +426,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) "you to." ) else: - stable_parts.append( + post_workspace_parts.append( f"Active Hermes profile: {active_profile}. This session reads " f"and writes {get_hermes_home()}/profiles/{active_profile}/. The default " f"profile's data lives at {get_hermes_home()}/skills/, {get_hermes_home()}/plugins/, " @@ -457,11 +472,16 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) if platform_key == "tui" and _effective_hint: _effective_hint = _tui_embedded_pane_clarifier(_effective_hint) if _effective_hint: - stable_parts.append(_effective_hint) + post_workspace_parts.append(_effective_hint) # ── Context tier (cwd-dependent, may change between sessions) ─ context_parts: List[str] = [] + if coding_workspace_parts: + context_parts.extend(coding_workspace_parts) + context_parts.extend(coding_trailing_parts) + context_parts.extend(post_workspace_parts) + # Note: ephemeral_system_prompt is NOT included here. It's injected at # API-call time only so it stays out of the cached/stored system prompt. if system_message is not None: @@ -523,6 +543,8 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) timestamp_line += f"\nModel: {agent.model}" if agent.provider: timestamp_line += f"\nProvider: {agent.provider}" + if agent.platform: + timestamp_line += f"\nPlatform: {agent.platform}" volatile_parts.append(timestamp_line) return { @@ -549,6 +571,7 @@ def build_system_prompt(agent: Any, system_message: Optional[str] = None) -> str """ parts = build_system_prompt_parts(agent, system_message=system_message) joined = "\n\n".join(p for p in (parts["stable"], parts["context"], parts["volatile"]) if p) + agent._cached_system_prompt_static = parts["stable"] # Surface context-file truncation warnings through the normal agent status # channel so gateway/CLI users see them in chat instead of only in logs. @@ -565,10 +588,65 @@ def invalidate_system_prompt(agent: Any) -> None: so the rebuilt prompt captures any writes from this session. """ agent._cached_system_prompt = None + agent._cached_system_prompt_static = None if agent._memory_store: agent._memory_store.load_from_disk() +def reconstruct_static_prefix( + agent: Any, + system_message: Optional[str] = None, + *, + log_label: str = "restore", +) -> None: + """Reconstruct ``_cached_system_prompt_static`` for a stored prompt. + + The static prefix is not persisted (only the full prompt is), so any + path that adopts a stored/kept ``_cached_system_prompt`` — session + restore, the compression keep-prompt path, or a failover to a cache-on + provider mid-turn (#72626) — must rebuild the stable tier to regain the + two-block ``[static, volatile]`` system layout. + + Safety: the rebuilt stable tier is used ONLY when the stored prompt + literally starts with it (checked here AND re-checked by + ``_apply_system_cache_markers``'s ``startswith`` gate). If any + stable-tier input changed since the prompt was persisted (skills + edited, identity changed), the prefix mismatches, the static stays + None, and requests fall back to the legacy layout with the stored + prompt bytes untouched — never a rewritten prompt. + + A failed reconstruction is memoized per stored prompt + (``_static_rebuild_failed_for``): ``build_system_prompt_parts`` does + real file I/O (SOUL.md, context files, memory), and callers on the + retry-loop hot path must not re-run it every attempt when the inputs + haven't changed. A legitimately changed stored prompt retries once. + """ + if not getattr(agent, "_use_prompt_caching", False): + return + stored = getattr(agent, "_cached_system_prompt", None) + if not isinstance(stored, str) or not stored: + return + existing = getattr(agent, "_cached_system_prompt_static", None) + if isinstance(existing, str) and existing and stored.startswith(existing): + return + if getattr(agent, "_static_rebuild_failed_for", None) == stored: + return + try: + static = build_system_prompt_parts(agent, system_message=system_message)["stable"] + if static and stored.startswith(static): + agent._cached_system_prompt_static = static + agent._static_rebuild_failed_for = None + return + except Exception: + logger.debug( + "static system-prefix reconstruction failed on %s", + log_label, + exc_info=True, + ) + agent._cached_system_prompt_static = None + agent._static_rebuild_failed_for = stored + + def format_tools_for_system_message(agent: Any) -> str: """Format tool definitions for the system message in the trajectory format. diff --git a/agent/title_generator.py b/agent/title_generator.py index 7469a665bfc5..80dcdd10c72c 100644 --- a/agent/title_generator.py +++ b/agent/title_generator.py @@ -43,10 +43,10 @@ def _title_language() -> str: """Return configured title language, or empty string to match the user.""" try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly return str( - ((load_config() or {}).get("auxiliary") or {}) + ((load_config_readonly() or {}).get("auxiliary") or {}) .get("title_generation", {}) .get("language", "") ).strip() @@ -71,6 +71,27 @@ def _auto_title_enabled() -> bool: return True +def _summarize_user_message(user_message: str) -> str: + """Collapse a slash-skill-expanded turn back to what the user typed. + + A ``/skill`` invocation expands into a message that embeds the whole skill + body, so feeding it to the titler verbatim titles the session after the + *skill's* prose — "Kick off a task in a fresh isolated git worktree" — not + after the user's request. Reuse the canonical scaffolding parser so the + model sees ``/work — fix the title leak`` instead. + """ + if not user_message: + return "" + try: + from agent.skill_commands import describe_skill_invocation + + described = describe_skill_invocation(user_message) + except Exception: + logger.debug("Skill-scaffolding summary failed; titling raw", exc_info=True) + return user_message + return described if described is not None else user_message + + def generate_title( user_message: str, assistant_response: str, @@ -110,7 +131,7 @@ def generate_title( logger.debug("Title runtime validator raised; proceeding", exc_info=True) # Truncate long messages to keep the request small - user_snippet = user_message[:500] if user_message else "" + user_snippet = _summarize_user_message(user_message)[:500] assistant_snippet = assistant_response[:500] if assistant_response else "" language = _title_language() @@ -143,6 +164,11 @@ def generate_title( title = title.strip('"\'') if title.lower().startswith("title:"): title = title[6:].strip() + # A title is one line. A model that ignores "return ONLY the title" and + # answers the prompt instead (a shell transcript, a bulleted plan) would + # otherwise be stored verbatim and truncated mid-command. Keep the first + # non-empty line — the closest thing to a title in that response. + title = next((line.strip() for line in title.splitlines() if line.strip()), "") # Enforce reasonable length if len(title) > 80: title = title[:77] + "..." diff --git a/agent/tool_dispatch_helpers.py b/agent/tool_dispatch_helpers.py index 07b2c2e65e79..f7f003f24abb 100644 --- a/agent/tool_dispatch_helpers.py +++ b/agent/tool_dispatch_helpers.py @@ -4,9 +4,10 @@ * ``_is_destructive_command`` — terminal-command heuristic used to gate parallel batch dispatch. -* ``_should_parallelize_tool_batch`` / ``_extract_parallel_scope_path`` / - ``_paths_overlap`` — the rules engine deciding when a multi-tool batch - can run concurrently. +* ``_should_parallelize_tool_batch`` / ``_extract_parallel_scope_paths`` / + ``_extract_parallel_scope_path`` / ``_paths_overlap`` — the rules engine + deciding when a multi-tool batch can run concurrently (V4A patch scope + uses patch-body file headers, not a decoy ``path=``). * ``_is_multimodal_tool_result`` / ``_multimodal_text_summary`` / ``_append_subdir_hint_to_multimodal`` — envelope helpers for the ``{"_multimodal": True, "content": [...], "text_summary": ...}`` dict @@ -57,8 +58,17 @@ "web_search", }) +# Filesystem tools whose parallel admission is decided by path overlap. +# Readers may share a subtree with other readers; a writer conflicts with +# ANY overlapping reservation (reader or writer). This is what keeps a +# batched ``search_files``/``read_file`` from observing pre-mutation file +# state when the model batches it alongside the ``patch``/``write_file`` +# it depends on (the classic same-block write→read race). +_PATH_SCOPED_READERS = frozenset({"read_file", "search_files"}) +_PATH_SCOPED_WRITERS = frozenset({"write_file", "patch"}) + # File tools can run concurrently when they target independent paths. -_PATH_SCOPED_TOOLS = frozenset({"read_file", "write_file", "patch"}) +_PATH_SCOPED_TOOLS = _PATH_SCOPED_READERS | _PATH_SCOPED_WRITERS # Patterns that indicate a terminal command may modify/delete files. _DESTRUCTIVE_PATTERNS = re.compile( @@ -116,10 +126,18 @@ def _plan_tool_batch_segments(tool_calls, *, execution_cwd: Optional[Path] = Non * ``_NEVER_PARALLEL_TOOLS`` (interactive tools) → barrier. * Unparseable / non-dict arguments → barrier. - * Path-scoped tools (``read_file``/``write_file``/``patch``) join a - parallel run only when their target path does not overlap another - path already reserved in the same run; an overlap closes the run so - the conflicting call starts a NEW run after the first completes. + * Path-scoped tools (``read_file``/``search_files``/``write_file``/ + ``patch``) join a parallel run only when their target path(s) do not + CONFLICT with a path already reserved in the same run. Reservations + carry a reader/writer role: reader↔reader overlap is harmless (two + reads of the same file commute) and stays parallel; any overlap + involving a writer closes the run so the conflicting call starts a + NEW run after the first completes. ``search_files`` reserves its + search root (default ``.``) as a reader — a search batched after a + write into the searched subtree is ordered behind that write instead + of racing it. For V4A ``patch(mode="patch")`` the reserved paths are + the file headers in the patch body, not a possibly-stale ``path=`` + argument. * Anything not in ``_PARALLEL_SAFE_TOOLS`` and not an opted-in MCP tool → barrier. @@ -129,7 +147,8 @@ def _plan_tool_batch_segments(tool_calls, *, execution_cwd: Optional[Path] = Non """ segments: list[list] = [] # [kind, calls] pairs, normalized to tuples on return current: list = [] - reserved_paths: list[Path] = [] + # (canonical_path, is_writer) reservations for the current parallel run. + reserved_paths: list[tuple[Path, bool]] = [] def _close_parallel() -> None: nonlocal current, reserved_paths @@ -173,15 +192,25 @@ def _add_sequential(tc) -> None: continue if tool_name in _PATH_SCOPED_TOOLS: - scoped_path = _extract_parallel_scope_path(tool_name, function_args, execution_cwd=execution_cwd) - if scoped_path is None: + scoped_paths = _extract_parallel_scope_paths( + tool_name, function_args, execution_cwd=execution_cwd + ) + if not scoped_paths: _add_sequential(tool_call) continue - if any(_paths_overlap(scoped_path, existing) for existing in reserved_paths): + is_writer = tool_name in _PATH_SCOPED_WRITERS + if any( + (is_writer or existing_is_writer) + and _paths_overlap(scoped_path, existing) + for scoped_path in scoped_paths + for existing, existing_is_writer in reserved_paths + ): # Same-subtree conflict inside this run: close it so this # call starts a fresh run AFTER the conflicting one lands. + # Reader↔reader overlap never conflicts — concurrent reads + # of the same subtree commute. _close_parallel() - reserved_paths.append(scoped_path) + reserved_paths.extend((p, is_writer) for p in scoped_paths) current.append(tool_call) continue @@ -233,33 +262,77 @@ def _canonical_path(raw_path: str, execution_cwd: Optional[Path] = None) -> Path return Path(resolved) -def _extract_parallel_scope_path( +def _extract_parallel_scope_paths( tool_name: str, function_args: dict, execution_cwd: Optional[Path] = None, -) -> Optional[Path]: - """Return the canonical file target for path-scoped tools. +) -> List[Path]: + """Return every canonical path this call reserves for overlap checks. *execution_cwd* should be the working directory that the tool will actually use at runtime. When omitted the process cwd is used, which may differ from the tool execution environment on some platforms (e.g. WSL, sandboxed sub-processes). + + For ``patch`` in V4A ``mode=patch``, scope comes from patch-body + ``*** Update/Add/Delete/Move File:`` headers (not a possibly-decoy + ``path=``). An empty result means the planner cannot determine the + scope and must treat the call as a sequential barrier. """ if tool_name not in _PATH_SCOPED_TOOLS: - return None + return [] - raw_path = function_args.get("path") - if not isinstance(raw_path, str) or not raw_path.strip(): - return None + raw_paths: List[str] = [] + if tool_name == "patch" and (function_args.get("mode") or "replace") == "patch": + raw_paths.extend(_extract_file_mutation_targets(tool_name, function_args)) + else: + raw_path = function_args.get("path") + if isinstance(raw_path, str) and raw_path.strip(): + raw_paths.append(raw_path) + elif tool_name == "search_files": + # ``search_files`` defaults its search root to the cwd when + # ``path`` is omitted — reserve that root rather than falling + # back to a sequential barrier (an empty result here would + # demote every bare search to a barrier and destroy read + # parallelism). + raw_paths.append(".") + + scoped: List[Path] = [] + seen: set[str] = set() + for raw in raw_paths: + if not isinstance(raw, str) or not raw.strip(): + continue + canonical = _canonical_path(raw, execution_cwd) + key = str(canonical) + if key in seen: + continue + seen.add(key) + scoped.append(canonical) + return scoped + + +def _extract_parallel_scope_path( + tool_name: str, + function_args: dict, + execution_cwd: Optional[Path] = None, +) -> Optional[Path]: + """Return the primary canonical file target for path-scoped tools. - return _canonical_path(raw_path, execution_cwd) + Thin view over ``_extract_parallel_scope_paths`` kept for callers/tests + that only need a single representative path. For multi-file V4A + patches this is the first header target. + """ + scoped = _extract_parallel_scope_paths( + tool_name, function_args, execution_cwd=execution_cwd + ) + return scoped[0] if scoped else None def _paths_overlap(left: Path, right: Path) -> bool: """Return True when two paths may refer to the same subtree. Both *left* and *right* must already be canonical (as returned by - ``_extract_parallel_scope_path`` / ``_canonical_path``) so that + ``_extract_parallel_scope_paths`` / ``_canonical_path``) so that symlink aliases and case differences are already normalised. """ left_parts = left.parts @@ -354,8 +427,10 @@ def _extract_file_mutation_targets(tool_name: str, args: Dict[str, Any]) -> List if not isinstance(body, str) or not body: return [] paths: List[str] = [] + # ``\s*`` (not ``\s+``) after ``***`` matches patch_parser / file_tools: + # they accept ``***Update File:`` with no space after the asterisks. for _m in re.finditer( - r'^\*\*\*\s+(?:Update|Add|Delete)\s+File:\s*(.+)$', + r'^\*\*\*\s*(?:Update|Add|Delete)\s+File:\s*(.+)$', body, re.MULTILINE, ): @@ -363,7 +438,7 @@ def _extract_file_mutation_targets(tool_name: str, args: Dict[str, Any]) -> List if p: paths.append(p) for _m in re.finditer( - r'^\*\*\*\s+Move\s+File:\s*(.+?)\s*->\s*(.+)$', + r'^\*\*\*\s*Move\s+File:\s*(.+?)\s*->\s*(.+)$', body, re.MULTILINE, ): @@ -634,6 +709,8 @@ def _maybe_wrap_untrusted(name: str, content: Any) -> Any: "_NEVER_PARALLEL_TOOLS", "_PARALLEL_SAFE_TOOLS", "_PATH_SCOPED_TOOLS", + "_PATH_SCOPED_READERS", + "_PATH_SCOPED_WRITERS", "_DESTRUCTIVE_PATTERNS", "_REDIRECT_OVERWRITE", "_is_destructive_command", @@ -641,6 +718,7 @@ def _maybe_wrap_untrusted(name: str, content: Any) -> Any: "_should_parallelize_tool_batch", "_canonical_path", "_extract_parallel_scope_path", + "_extract_parallel_scope_paths", "_paths_overlap", "_is_multimodal_tool_result", "_multimodal_text_summary", diff --git a/agent/tool_executor.py b/agent/tool_executor.py index d235de36c03d..c422c118e727 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -20,6 +20,7 @@ import random import threading import time +from dataclasses import dataclass from typing import Any, Optional from agent.display import ( @@ -31,7 +32,6 @@ redact_tool_args_for_display as _redact_tool_args_for_display, _detect_tool_failure, ) -from agent.tool_guardrails import ToolGuardrailDecision from agent.tool_dispatch_helpers import ( _is_destructive_command, _is_multimodal_tool_result, @@ -140,8 +140,8 @@ def _flush_session_db_after_tool_progress( messages: list, *, stage: str, -) -> None: - """Best-effort incremental SessionDB flush for tool-call progress. +) -> bool: + """Flush tool-call progress before projecting it to any UI surface. Tool execution can perform side effects that terminate or restart the current Hermes process before the normal turn-end persistence path runs. @@ -149,9 +149,14 @@ def _flush_session_db_after_tool_progress( transcript survives destructive-but-valid tool calls. """ try: - agent._flush_messages_to_session_db(messages) + persisted = agent._flush_messages_to_session_db(messages) is not False + if not persisted: + agent._incremental_persistence_failed = True + return persisted except Exception as exc: + agent._incremental_persistence_failed = True logger.warning("Incremental tool-call persistence failed after %s: %s", stage, exc) + return False def _ra(): @@ -287,63 +292,338 @@ def _tool_search_scoped_names(agent) -> frozenset: return names -def _apply_tool_request_middleware_for_agent( +@dataclass +class _ManagedToolResult: + result: Any + args: dict[str, Any] + middleware_trace: list[dict[str, Any]] + blocked: bool + + +class _ConcurrentToolAuthorizationGate: + """Serialize policy prompts and exclude their queue from batch deadlines.""" + + def __init__(self) -> None: + self._serialization_lock = threading.Lock() + self._state_lock = threading.Lock() + self._pending = 0 + self._window_started: float | None = None + self._excluded_seconds = 0.0 + + def run(self, callback): + now = time.monotonic() + with self._state_lock: + if self._pending == 0: + self._window_started = now + self._pending += 1 + try: + with self._serialization_lock: + return callback() + finally: + now = time.monotonic() + with self._state_lock: + self._pending -= 1 + if self._pending == 0: + if self._window_started is not None: + self._excluded_seconds += max( + 0.0, now - self._window_started + ) + self._window_started = None + + def excluded_seconds(self) -> float: + """Return completed plus currently active authorization wait time.""" + now = time.monotonic() + with self._state_lock: + excluded = self._excluded_seconds + if self._window_started is not None: + excluded += max(0.0, now - self._window_started) + return excluded + + +def _managed_values( + outcome: _ManagedToolResult, +) -> tuple[Any, dict[str, Any], list[dict[str, Any]], bool]: + return ( + outcome.result, + outcome.args, + outcome.middleware_trace, + outcome.blocked, + ) + + +def _run_agent_tool_execution_middleware( agent, *, function_name: str, function_args: dict, effective_task_id: str, tool_call_id: str, -) -> tuple[dict, list[dict[str, Any]]]: - try: - from hermes_cli.middleware import apply_tool_request_middleware + execute, + scope_block: str | None = None, + display_index: int | None = None, + middleware_trace: list[dict[str, Any]] | None = None, + begin_execution=None, + authorization_gate: _ConcurrentToolAuthorizationGate | None = None, +) -> _ManagedToolResult: + """Run Relay rewrites before Hermes policy and dispatch exactly once.""" + from agent import relay_tools + from hermes_cli.middleware import ( + apply_tool_request_middleware, + run_tool_execution_middleware, + ) - result = apply_tool_request_middleware( + trace = middleware_trace if middleware_trace is not None else [] + state = { + "args": function_args, + "middleware_trace": trace, + "blocked": False, + "dispatched": False, + } + dispatch_lock = threading.Lock() + + def _authorized_dispatch(final_args: dict[str, Any]) -> Any: + with dispatch_lock: + if state["dispatched"]: + raise RuntimeError( + "Hermes tool execution callback invoked more than once" + ) + state["dispatched"] = True + state["blocked"] = False + state["args"] = final_args + + def _begin() -> None: + _begin_tool_execution( + agent, + function_name=function_name, + function_args=final_args, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + display_index=display_index, + ) + + def _advance_start_order(callback=None) -> None: + if begin_execution is None: + if callback is not None: + callback() + return + begin_execution(callback) + + block_message = scope_block + block_error_type = "tool_scope_block" + if block_message is None: + block_error_type = "plugin_block" + + def _resolve_pre_tool_block(): + try: + from hermes_cli.plugins import resolve_pre_tool_block + + return resolve_pre_tool_block( + function_name, + final_args, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + middleware_trace=list(state["middleware_trace"]), + ) + except Exception: + return None + + block_message = ( + _resolve_pre_tool_block() + if authorization_gate is None + else authorization_gate.run(_resolve_pre_tool_block) + ) + + guardrail_decision = None + if block_message is None: + guardrail_decision = agent._tool_guardrails.before_call( + function_name, final_args + ) + if guardrail_decision.allows_execution: + guardrail_decision = None + + if block_message is not None or guardrail_decision is not None: + _advance_start_order() + state["blocked"] = True + if block_message is not None: + result = json.dumps({"error": block_message}, ensure_ascii=False) + error_type = block_error_type + error_message = block_message + else: + result = agent._guardrail_block_result(guardrail_decision) + error_type = "guardrail_block" + error_message = ( + getattr(guardrail_decision, "message", None) + or "Tool blocked by guardrail policy" + ) + _emit_terminal_post_tool_call( + agent, + function_name=function_name, + function_args=final_args, + result=result, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + status="blocked", + error_type=error_type, + error_message=error_message, + middleware_trace=list(state["middleware_trace"]), + ) + return result + + if function_name == "memory": + agent._turns_since_memory = 0 + elif function_name == "skill_manage": + agent._iters_since_skill = 0 + + _advance_start_order(_begin) + return execute(final_args) + + def _hermes_pipeline(relay_args: dict[str, Any]) -> Any: + request_result = apply_tool_request_middleware( function_name, - function_args, + relay_args, + skip_relay=True, + task_id=effective_task_id or "", + session_id=getattr(agent, "session_id", "") or "", + tool_call_id=tool_call_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") or "", + ) + request_args = ( + request_result.payload + if isinstance(request_result.payload, dict) + else relay_args + ) + trace.clear() + trace.extend(request_result.trace) + return run_tool_execution_middleware( + function_name, + request_args, + lambda next_args: _authorized_dispatch( + next_args if isinstance(next_args, dict) else request_args + ), + original_args=function_args, task_id=effective_task_id or "", session_id=getattr(agent, "session_id", "") or "", tool_call_id=tool_call_id or "", turn_id=getattr(agent, "_current_turn_id", "") or "", api_request_id=getattr(agent, "_current_api_request_id", "") or "", ) - payload = result.payload if isinstance(result.payload, dict) else function_args - return payload, list(result.trace) - except Exception as exc: - logger.debug("tool_request middleware error: %s", exc) - return function_args, [] + result, _relay_args = relay_tools.execute( + function_name, + function_args, + _hermes_pipeline, + session_id=str(getattr(agent, "session_id", "") or ""), + metadata={ + "task_id": effective_task_id or "", + "turn_id": getattr(agent, "_current_turn_id", "") or "", + "api_request_id": getattr(agent, "_current_api_request_id", "") or "", + "tool_call_id": tool_call_id or "", + }, + ) + return _ManagedToolResult( + result=result, + args=state["args"], + middleware_trace=state["middleware_trace"], + blocked=bool(state["blocked"]), + ) -def _run_agent_tool_execution_middleware( + +def _begin_tool_execution( agent, *, function_name: str, - function_args: dict, + function_args: dict[str, Any], effective_task_id: str, tool_call_id: str, - execute, -) -> tuple[Any, dict]: - observed_args = function_args + display_index: int | None, +) -> None: + """Run user-visible and checkpoint preflight on final tool arguments.""" + if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": + display_args = ( + _redact_tool_args_for_display(function_name, function_args) or function_args + ) + args_str = json.dumps(display_args, ensure_ascii=False) + prefix = f"Tool {display_index}" if display_index is not None else "Tool" + if agent.verbose_logging: + print(f" 📞 {prefix}: {function_name}({list(display_args.keys())})") + print( + agent._wrap_verbose( + "Args: ", json.dumps(display_args, indent=2, ensure_ascii=False) + ) + ) + else: + args_preview = ( + args_str[: agent.log_prefix_chars] + "..." + if len(args_str) > agent.log_prefix_chars + else args_str + ) + print( + f" 📞 {prefix}: {function_name}({list(function_args.keys())}) - " + f"{args_preview}" + ) - def _execute(next_args: dict) -> Any: - nonlocal observed_args - observed_args = next_args if isinstance(next_args, dict) else function_args - return execute(observed_args) + agent._current_tool = function_name + agent._touch_activity(f"executing tool: {function_name}") + try: + from tools.environments.base import set_activity_callback - from hermes_cli.middleware import run_tool_execution_middleware + set_activity_callback(agent._touch_activity) + except Exception: + pass - result = run_tool_execution_middleware( - function_name, - function_args, - _execute, - original_args=function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=tool_call_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - ) - return result, observed_args + if agent.tool_progress_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + preview = _build_tool_preview(function_name, display_args) + agent.tool_progress_callback( + "tool.started", function_name, preview, display_args + ) + except Exception as callback_error: + logging.debug("Tool progress callback error: %s", callback_error) + + if agent.tool_start_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + agent.tool_start_callback( + tool_call_id, function_name, display_args + ) + except Exception as callback_error: + logging.debug("Tool start callback error: %s", callback_error) + + if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: + try: + _ensure_file_checkpoint( + agent, + function_name, + function_args, + effective_task_id, + ) + except Exception: + pass + + if function_name == "terminal" and agent._checkpoint_mgr.enabled: + try: + command = function_args.get("command", "") + if _is_destructive_command(command): + cwd = function_args.get("workdir") or os.getenv( + "TERMINAL_CWD", os.getcwd() + ) + agent._checkpoint_mgr.ensure_checkpoint( + cwd, f"before terminal: {command[:60]}" + ) + except Exception: + pass def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: @@ -381,7 +661,9 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe return # ── Parse args + pre-execution bookkeeping ─────────────────────── - parsed_calls = [] # list of (tool_call, function_name, function_args, middleware_trace, block_result, blocked_by_guardrail) + # (tool call, resolved name, parsed args, middleware trace, parse error, + # tool-search scope block) + parsed_calls = [] for tool_call in tool_calls: function_name = tool_call.function.name @@ -397,17 +679,11 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe function_args, [], malformed_args_result, - False, + None, ) ) continue - # Reset nudge counters only for a structurally valid invocation. - if function_name == "memory": - agent._turns_since_memory = 0 - elif function_name == "skill_manage": - agent._iters_since_skill = 0 - # ── Tool Search unwrap ──────────────────────────────────────── # When the model invokes the tool_call bridge, peel it open so # every downstream check (checkpointing, guardrails, plugin @@ -431,170 +707,68 @@ def execute_tool_calls_concurrent(agent, assistant_message, messages: list, effe _underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args) if not _err and _underlying: if _underlying in _tool_search_scoped_names(agent): - function_name = _underlying - function_args = _underlying_args + # Probe-validate before unwrapping (ironclaw#5149): + # missing required args return the parameter schema + # instead of dispatching into an opaque failure. + _probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args) + if _probe_err is not None: + _ts_scope_block = _probe_err + else: + function_name = _underlying + function_args = _underlying_args else: - _ts_scope_block = json.dumps({ - "error": ( - f"'{_underlying}' is not available in this session. " - "Use tool_search to find tools you can call." - ), - }, ensure_ascii=False) + _ts_scope_block = ( + f"'{_underlying}' is not available in this session. " + "Use tool_search to find tools you can call." + ) except Exception: pass - function_args, middleware_trace = _apply_tool_request_middleware_for_agent( - agent, - function_name=function_name, - function_args=function_args, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", + parsed_calls.append( + (tool_call, function_name, function_args, [], None, _ts_scope_block) ) - # ── Block evaluation (BEFORE checkpoint preflight) ─────────── - # We must know whether the tool will execute before touching - # checkpoint state (dedup slot, real snapshots). - block_result = None - blocked_by_guardrail = False - if _ts_scope_block is not None: - # Out-of-scope tool_call: reject before hooks/guardrails/dispatch. - block_result = _ts_scope_block - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="tool_scope_block", - error_message=_ts_scope_block, - middleware_trace=list(middleware_trace), - ) - else: - try: - from hermes_cli.plugins import resolve_pre_tool_block - block_message = resolve_pre_tool_block( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=getattr(tool_call, "id", "") or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - middleware_trace=list(middleware_trace), - ) - except Exception: - block_message = None - - if block_message is not None: - block_result = json.dumps({"error": block_message}, ensure_ascii=False) - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="plugin_block", - error_message=block_message, - middleware_trace=list(middleware_trace), - ) - else: - guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - block_result = agent._guardrail_block_result(guardrail_decision) - blocked_by_guardrail = True - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=block_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="guardrail_block", - error_message=getattr(guardrail_decision, "message", None) or "Tool blocked by guardrail policy", - middleware_trace=list(middleware_trace), - ) - - # ── Checkpoint preflight (only for tools that will execute) ── - if block_result is None: - # Checkpoint for file-mutating tools - if function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: - try: - _ensure_file_checkpoint( - agent, - function_name, - function_args, - effective_task_id, - ) - except Exception: - pass - - # Checkpoint before destructive terminal commands - if function_name == "terminal" and agent._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - agent._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass - - parsed_calls.append((tool_call, function_name, function_args, middleware_trace, block_result, blocked_by_guardrail)) - # ── Logging / callbacks ────────────────────────────────────────── tool_names_str = ", ".join(name for _, name, _, _, _, _ in parsed_calls) if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}") - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls, 1): - display_args = _redact_tool_args_for_display(name, args) or args - args_str = json.dumps(display_args, ensure_ascii=False) - if agent.verbose_logging: - print(f" 📞 Tool {i}: {name}({list(display_args.keys())})") - print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {name}({list(args.keys())}) - {args_preview}") - - for tc, name, args, middleware_trace, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if agent.tool_progress_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - preview = _build_tool_preview(name, display_args) - agent.tool_progress_callback("tool.started", name, preview, display_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - for tc, name, args, middleware_trace, block_result, blocked_by_guardrail in parsed_calls: - if block_result is not None: - continue - if agent.tool_start_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - agent.tool_start_callback(tc.id, name, display_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") # ── Concurrent execution ───────────────────────────────────────── # Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag, middleware_trace) results = [None] * num_tools - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + for i, (tc, name, args, middleware_trace, block_result, _scope_block) in enumerate(parsed_calls): if block_result is not None: results[i] = (name, args, block_result, 0.0, True, True, middleware_trace) + start_condition = threading.Condition() + next_start_order = 0 + authorization_gate = _ConcurrentToolAuthorizationGate() + + def _begin_in_order(order: int, callback=None) -> None: + nonlocal next_start_order + with start_condition: + start_condition.wait_for(lambda: order == next_start_order) + try: + if callback is not None: + callback() + finally: + next_start_order += 1 + start_condition.notify_all() + # Touch activity before launching workers so the gateway knows # we're executing tools (not stuck). agent._current_tool = tool_names_str agent._touch_activity(f"executing {num_tools} tools concurrently: {tool_names_str}") - def _run_tool(index, tool_call, function_name, function_args, middleware_trace): + def _run_tool( + index, + tool_call, + function_name, + function_args, + middleware_trace, + scope_block, + start_order, + ): """Worker function executed in a thread.""" # Register this worker tid so the agent can fan out an interrupt # to it — see AIAgent.interrupt(). Must happen first thing, and @@ -624,18 +798,50 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): # ContextVars are propagated by propagate_context_to_thread() at the # submit site below (GHSA-qg5c-hvr5-hjgr, #13617). start = time.time() + blocked = False + start_advanced = False + + def _advance_start(callback=None) -> None: + nonlocal start_advanced + if start_advanced: + return + try: + _begin_in_order(start_order, callback) + finally: + start_advanced = True + try: try: - result = agent._invoke_tool( - function_name, - function_args, - effective_task_id, - tool_call.id, - messages=messages, - pre_tool_block_checked=True, - skip_tool_request_middleware=True, - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict[str, Any]) -> Any: + return agent._invoke_tool( + function_name, + next_args, + effective_task_id, + tool_call.id, + messages=messages, + pre_tool_block_checked=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + ) + + managed = _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=scope_block, + display_index=index + 1, + middleware_trace=middleware_trace, + begin_execution=_advance_start, + authorization_gate=authorization_gate, ) + result = managed.result + function_args = managed.args + middleware_trace = managed.middleware_trace + blocked = managed.blocked except KeyboardInterrupt: try: agent.interrupt("keyboard interrupt") @@ -652,7 +858,15 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): ) duration = time.time() - start logger.info("tool %s cancelled (%.2fs)", function_name, duration) - results[index] = (function_name, function_args, result, duration, True, False, middleware_trace) + results[index] = ( + function_name, + function_args, + result, + duration, + True, + False, + middleware_trace, + ) return except Exception as tool_error: result = f"Error executing tool '{function_name}': {tool_error}" @@ -663,8 +877,17 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200]) else: logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result)) - results[index] = (function_name, function_args, result, duration, is_error, False, middleware_trace) + results[index] = ( + function_name, + function_args, + result, + duration, + is_error, + blocked, + middleware_trace, + ) finally: + _advance_start() # Tear down worker-tid tracking. Clear any interrupt bit we may # have set so the next task scheduled onto this recycled tid # starts with a clean slate. This MUST be in a finally block @@ -687,9 +910,11 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): try: runnable_calls = [ - (i, tc, name, args) - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls) - if block_result is None + (i, tc, name, args, scope_block) + for i, (tc, name, args, _trace, parse_error, scope_block) in enumerate( + parsed_calls + ) + if parse_error is None ] futures = [] future_to_index = {} @@ -707,13 +932,22 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): executor = DaemonThreadPoolExecutor(max_workers=max_workers) abandon_executor = False try: - for submit_index, (i, tc, name, args) in enumerate(runnable_calls): + for submit_index, (i, tc, name, args, scope_block) in enumerate( + runnable_calls + ): # Propagate the agent turn's ContextVars (e.g. # _approval_session_key) AND thread-local approval/sudo # callbacks into the worker thread; clears callbacks on exit. try: f = executor.submit( - propagate_context_to_thread(_run_tool), i, tc, name, args, parsed_calls[i][3] + propagate_context_to_thread(_run_tool), + i, + tc, + name, + args, + parsed_calls[i][3], + scope_block, + submit_index, ) except RuntimeError as submit_error: if not _is_interpreter_shutdown_submit_error(submit_error): @@ -724,7 +958,13 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): "skipping %d unsubmitted tool(s)", len(skipped_calls), ) - for skipped_i, _tc, skipped_name, skipped_args in skipped_calls: + for ( + skipped_i, + _tc, + skipped_name, + skipped_args, + _scope_block, + ) in skipped_calls: if results[skipped_i] is None: middleware_trace = parsed_calls[skipped_i][3] result = ( @@ -754,7 +994,10 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): while True: wait_timeout = 5.0 if deadline is not None: - remaining = deadline - time.monotonic() + effective_deadline = ( + deadline + authorization_gate.excluded_seconds() + ) + remaining = effective_deadline - time.monotonic() if remaining <= 0: done, not_done = set(), { f for f in futures if not f.done() @@ -771,7 +1014,11 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): if not not_done: break - if deadline is not None and time.monotonic() >= deadline: + if ( + deadline is not None + and time.monotonic() + >= deadline + authorization_gate.excluded_seconds() + ): abandon_executor = True timed_out_indices = { future_to_index[f] @@ -851,9 +1098,13 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total") # ── Post-execution: display per-tool results ───────────────────── - for i, (tc, name, args, middleware_trace, block_result, blocked_by_guardrail) in enumerate(parsed_calls): + for i, (tc, name, args, middleware_trace, _parse_error, _scope_block) in enumerate( + parsed_calls + ): r = results[i] blocked = False + is_error = True + progress_function_name = name # A worker can finish and write results[i] in the window between the # deadline snapshot (timed_out_indices, taken from not_done) and this # loop. Prefer that real result over a fabricated timeout message — the @@ -909,6 +1160,9 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): tool_duration = 0.0 else: function_name, function_args, function_result, tool_duration, is_error, blocked, middleware_trace = r + name = function_name + args = function_args + progress_function_name = function_name if blocked: effect_disposition = "none" @@ -936,43 +1190,15 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): except Exception as _ver_err: logging.debug("file-mutation verifier record failed: %s", _ver_err) - if not blocked and agent.tool_progress_callback: - try: - agent.tool_progress_callback( - "tool.completed", function_name, None, None, - duration=tool_duration, is_error=is_error, - result=function_result, - ) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - if agent.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") - - # Print cute message per tool - if agent._should_emit_quiet_tool_messages(): - cute_msg = _get_cute_tool_message_impl(name, args, tool_duration, result=function_result) - agent._safe_print(f" {cute_msg}") - elif not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": - _preview_str = _multimodal_text_summary(function_result) if agent.verbose_logging: - print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s") - print(agent._wrap_verbose("Result: ", _preview_str)) - else: - response_preview = _preview_str[:agent.log_prefix_chars] + "..." if len(_preview_str) > agent.log_prefix_chars else _preview_str - print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") + logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) + logging.debug("Tool result (%d chars): %s", len(function_result), function_result) agent._current_tool = None - agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)") - - if not blocked and agent.tool_complete_callback: - try: - display_args = _redact_tool_args_for_display(name, args) or args - agent.tool_complete_callback(tc.id, name, display_args, function_result) - except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") + _status_suffix = " (error)" if is_error else "" + agent._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s){_status_suffix}") + display_function_result = function_result function_result = maybe_persist_tool_result( content=function_result, tool_name=name, @@ -1007,6 +1233,50 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): ) messages.append(tool_message) risk_metadata = tool_message.get("_tool_output_risk") + if not _flush_session_db_after_tool_progress( + agent, + messages, + stage=f"tool result {name}", + ): + return + + # Every completion surface is downstream of the canonical append. If + # the UI bridge or process dies while projecting one of these events, + # resume can reconstruct the tool result that was already visible. + if not blocked and agent.tool_progress_callback: + try: + agent.tool_progress_callback( + "tool.completed", progress_function_name, None, None, + duration=tool_duration, is_error=is_error, + result=display_function_result, + ) + except Exception as cb_err: + logging.debug("Tool progress callback error: %s", cb_err) + + # Print cute message per tool + if agent._should_emit_quiet_tool_messages(): + cute_msg = _get_cute_tool_message_impl( + name, args, tool_duration, result=display_function_result, + ) + agent._safe_print(f" {cute_msg}") + elif not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": + _preview_str = _multimodal_text_summary(display_function_result) + if agent.verbose_logging: + print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s") + print(agent._wrap_verbose("Result: ", _preview_str)) + else: + response_preview = _preview_str[:agent.log_prefix_chars] + "..." if len(_preview_str) > agent.log_prefix_chars else _preview_str + print(f" ✅ Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") + + if not blocked and agent.tool_complete_callback: + try: + display_args = _redact_tool_args_for_display(name, args) or args + agent.tool_complete_callback( + tc.id, name, display_args, display_function_result, + ) + except Exception as cb_err: + logging.debug("Tool complete callback error: %s", cb_err) + if ( risk_metadata is not None and risk_metadata.get("risk") != "low" @@ -1023,18 +1293,11 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): ) except Exception as cb_err: logging.debug("Tool output risk callback error: %s", cb_err) - _flush_session_db_after_tool_progress( - agent, - messages, - stage=f"tool result {name}", - ) - - # ── Per-tool /steer drain ─────────────────────────────────── - # Same as the sequential path: drain between each collected - # result so the steer lands as early as possible. - agent._apply_pending_steer_to_tool_results(messages, 1) # ── Per-turn aggregate budget enforcement ───────────────────────── + # Keep /steer pending until the final post-budget drain below. The model + # cannot observe a partial batch, while an early drain can be discarded + # when aggregate budget enforcement replaces that tool result. num_tools = len(parsed_calls) if finalize and num_tools > 0: turn_tool_msgs = messages[-num_tools:] @@ -1049,6 +1312,26 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace): +def _append_cancelled_tool_results(messages: list, tool_calls, *, reason: str) -> None: + """Append a cancelled ``tool`` result for each call in ``tool_calls``. + + Used when a hard interrupt (KeyboardInterrupt / BaseException) aborts the + sequential executor mid-batch. Without this, the loop re-raises leaving the + assistant tool-call turn with no matching tool results — a message-role + alternation violation that malforms the next provider request. Mirrors the + cooperative-interrupt skip block and the concurrent path, both of which + already emit a result for every call_id. + """ + for tc in tool_calls: + name = getattr(getattr(tc, "function", None), "name", "") or "tool" + messages.append(make_tool_result_message( + name, + f"[Tool execution cancelled — {name} was skipped due to {reason}]", + getattr(tc, "id", "") or "", + effect_disposition="none", + )) + + def execute_tool_calls_sequential(agent, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0, *, finalize: bool = True) -> None: """Execute tool calls sequentially (original behavior). Used for single calls or interactive tools. @@ -1059,6 +1342,8 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe # Resolve the context-scaled tool-output budget once per turn. _tool_budget = _budget_for_agent(agent) for i, tool_call in enumerate(assistant_message.tool_calls, 1): + if getattr(agent, "_incremental_persistence_failed", False): + return # SAFETY: check interrupt BEFORE starting each tool. # If the user sent "stop" during a previous tool's execution, # do NOT start any more tools -- skip them all immediately. @@ -1074,11 +1359,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe skipped_tc.id, effect_disposition="none", )) - _flush_session_db_after_tool_progress( + if not _flush_session_db_after_tool_progress( agent, messages, stage=f"cancelled tool result {skipped_name}", - ) + ): + return break function_name = tool_call.function.name @@ -1094,12 +1380,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe tool_call.id, ) ) - _flush_session_db_after_tool_progress( + if not _flush_session_db_after_tool_progress( agent, messages, stage=f"invalid tool arguments {function_name}", - ) - agent._apply_pending_steer_to_tool_results(messages, 1) + ): + return continue # Tool Search unwrap — see execute_tool_calls_concurrent for full @@ -1112,8 +1398,25 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe _underlying, _underlying_args, _err = _ts.resolve_underlying_call(function_args) if not _err and _underlying: if _underlying in _tool_search_scoped_names(agent): - function_name = _underlying - function_args = _underlying_args + # Probe-validate before unwrapping (ironclaw#5149): + # missing required args return the parameter schema + # instead of dispatching into an opaque failure. + _probe_err = _ts.validate_deferred_call_args(_underlying, _underlying_args) + if _probe_err is not None: + # This path wraps _block_msg in {"error": ...} — + # flatten the probe payload to one plain string. + try: + _probe = json.loads(_probe_err) + _ts_scope_block = ( + f"{_probe.get('error', '')} Parameters schema: " + f"{json.dumps(_probe.get('parameters', {}), ensure_ascii=False)}. " + f"{_probe.get('hint', '')}" + ).strip() + except Exception: + _ts_scope_block = _probe_err + else: + function_name = _underlying + function_args = _underlying_args else: _ts_scope_block = ( f"'{_underlying}' is not available in this session. " @@ -1122,153 +1425,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe except Exception: pass - function_args, middleware_trace = _apply_tool_request_middleware_for_agent( - agent, - function_name=function_name, - function_args=function_args, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - ) - - # Check plugin hooks for a block directive before executing. - _block_msg: Optional[str] = None - _block_error_type = "plugin_block" - if _ts_scope_block is not None: - _block_msg = _ts_scope_block - _block_error_type = "tool_scope_block" - else: - try: - from hermes_cli.plugins import resolve_pre_tool_block - _block_msg = resolve_pre_tool_block( - function_name, - function_args, - task_id=effective_task_id or "", - session_id=getattr(agent, "session_id", "") or "", - tool_call_id=getattr(tool_call, "id", "") or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - middleware_trace=list(middleware_trace), - ) - except Exception: - pass - - _guardrail_block_decision: ToolGuardrailDecision | None = None - if _block_msg is None: - guardrail_decision = agent._tool_guardrails.before_call(function_name, function_args) - if not guardrail_decision.allows_execution: - _guardrail_block_decision = guardrail_decision - - _execution_blocked = _block_msg is not None or _guardrail_block_decision is not None - - if _execution_blocked: - # Tool blocked by plugin or guardrail policy — skip counters, - # callbacks, checkpointing, activity mutation, and real execution. - pass - # Reset nudge counters when the relevant tool is actually used - elif function_name == "memory": - agent._turns_since_memory = 0 - elif function_name == "skill_manage": - agent._iters_since_skill = 0 - - if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - args_str = json.dumps(display_args, ensure_ascii=False) - if agent.verbose_logging: - print(f" 📞 Tool {i}: {function_name}({list(display_args.keys())})") - print(agent._wrap_verbose("Args: ", json.dumps(display_args, indent=2, ensure_ascii=False))) - else: - args_preview = args_str[:agent.log_prefix_chars] + "..." if len(args_str) > agent.log_prefix_chars else args_str - print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())}) - {args_preview}") - - if not _execution_blocked: - agent._current_tool = function_name - agent._touch_activity(f"executing tool: {function_name}") - - # Set activity callback for long-running tool execution (terminal - # commands, etc.) so the gateway's inactivity monitor doesn't kill - # the agent while a command is running. - if not _execution_blocked: - try: - from tools.environments.base import set_activity_callback - set_activity_callback(agent._touch_activity) - except Exception: - pass - - if not _execution_blocked and agent.tool_progress_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - preview = _build_tool_preview(function_name, display_args) - agent.tool_progress_callback("tool.started", function_name, preview, display_args) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - - if not _execution_blocked and agent.tool_start_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - agent.tool_start_callback(tool_call.id, function_name, display_args) - except Exception as cb_err: - logging.debug(f"Tool start callback error: {cb_err}") - - # Checkpoint: snapshot working dir before file-mutating tools - if not _execution_blocked and function_name in {"write_file", "patch"} and agent._checkpoint_mgr.enabled: - try: - _ensure_file_checkpoint( - agent, - function_name, - function_args, - effective_task_id, - ) - except Exception: - pass # never block tool execution - - # Checkpoint before destructive terminal commands - if not _execution_blocked and function_name == "terminal" and agent._checkpoint_mgr.enabled: - try: - cmd = function_args.get("command", "") - if _is_destructive_command(cmd): - cwd = function_args.get("workdir") or os.getenv("TERMINAL_CWD", os.getcwd()) - agent._checkpoint_mgr.ensure_checkpoint( - cwd, f"before terminal: {cmd[:60]}" - ) - except Exception: - pass # never block tool execution + middleware_trace: list[dict[str, Any]] = [] + _execution_blocked = False tool_start_time = time.time() - if _block_msg is not None: - # Tool blocked by plugin policy — return error without executing. - function_result = json.dumps({"error": _block_msg}, ensure_ascii=False) - tool_duration = 0.0 - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=function_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type=_block_error_type, - error_message=_block_msg, - middleware_trace=list(middleware_trace), - ) - elif _guardrail_block_decision is not None: - # Tool blocked by tool-loop guardrail — synthesize exactly one - # tool result for the original tool_call_id without executing. - function_result = agent._guardrail_block_result(_guardrail_block_decision) - tool_duration = 0.0 - _emit_terminal_post_tool_call( - agent, - function_name=function_name, - function_args=function_args, - result=function_result, - effective_task_id=effective_task_id, - tool_call_id=getattr(tool_call, "id", "") or "", - status="blocked", - error_type="guardrail_block", - error_message=getattr(_guardrail_block_decision, "message", None) or "Tool blocked by guardrail policy", - middleware_trace=list(middleware_trace), - ) - elif function_name == "todo": + if function_name == "todo": def _execute(next_args: dict) -> Any: from tools.todo_tool import todo_tool as _todo_tool return _todo_tool( @@ -1276,14 +1438,16 @@ def _execute(next_args: dict) -> Any: merge=next_args.get("merge", False), store=agent._todo_store, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('todo', function_args, tool_duration, result=function_result)}") @@ -1305,14 +1469,16 @@ def _execute(next_args: dict) -> Any: db=session_db, current_session_id=agent.session_id, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('session_search', function_args, tool_duration, result=function_result)}") @@ -1342,14 +1508,16 @@ def _execute(next_args: dict) -> Any: ), ) return result - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('memory', function_args, tool_duration, result=function_result)}") @@ -1359,16 +1527,19 @@ def _execute(next_args: dict) -> Any: return _clarify_tool( question=next_args.get("question", ""), choices=next_args.get("choices"), + multi_select=next_args.get("multi_select", False), callback=agent.clarify_callback, ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('clarify', function_args, tool_duration, result=function_result)}") @@ -1380,14 +1551,16 @@ def _execute(next_args: dict) -> Any: count=next_args.get("count"), callback=getattr(agent, "read_terminal_callback", None), ) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}") @@ -1412,14 +1585,16 @@ def _execute(next_args: dict) -> Any: try: def _execute(next_args: dict) -> Any: return agent._dispatch_delegate_task(next_args) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _delegate_result = function_result finally: agent._delegate_spinner = None @@ -1443,14 +1618,16 @@ def _execute(next_args: dict) -> Any: try: def _execute(next_args: dict) -> Any: return agent.context_compressor.handle_tool_call(function_name, next_args, messages=messages) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _ce_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Context engine tool '{function_name}' failed: {tool_error}"}) @@ -1477,14 +1654,16 @@ def _execute(next_args: dict) -> Any: try: def _execute(next_args: dict) -> Any: return agent._memory_manager.handle_tool_call(function_name, next_args) - function_result, function_args = _run_agent_tool_execution_middleware( + function_result, function_args, middleware_trace, _execution_blocked = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, function_args=function_args, effective_task_id=effective_task_id, tool_call_id=getattr(tool_call, "id", "") or "", execute=_execute, - ) + scope_block=_ts_scope_block, + display_index=i, + )) _mem_result = function_result except Exception as tool_error: function_result = json.dumps({"error": f"Memory tool '{function_name}' failed: {tool_error}"}) @@ -1507,18 +1686,46 @@ def _execute(next_args: dict) -> Any: spinner.start() _spinner_result = None try: - function_result = _ra().handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict) -> Any: + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) + + ( + function_result, + function_args, + middleware_trace, + _execution_blocked, + ) = _managed_values( + _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + middleware_trace=middleware_trace, + ) ) _spinner_result = function_result except KeyboardInterrupt: @@ -1536,6 +1743,14 @@ def _execute(next_args: dict) -> Any: agent.interrupt("keyboard interrupt") except Exception: pass + # Emit a tool result for THIS call and every remaining call in + # the batch before re-raising, so the assistant tool-call turn + # is never left without matching tool results (alternation). + _append_cancelled_tool_results( + messages, + assistant_message.tool_calls[i - 1:], + reason="keyboard interrupt", + ) raise except Exception as tool_error: function_result = f"Error executing tool '{function_name}': {tool_error}" @@ -1549,18 +1764,46 @@ def _execute(next_args: dict) -> Any: agent._vprint(f" {cute_msg}") else: try: - function_result = _ra().handle_function_call( - function_name, function_args, effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") or "", - enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None, - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - tool_request_middleware_trace=list(middleware_trace), + def _execute(next_args: dict) -> Any: + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) + + ( + function_result, + function_args, + middleware_trace, + _execution_blocked, + ) = _managed_values( + _run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + middleware_trace=middleware_trace, + ) ) except KeyboardInterrupt: _emit_cancelled_terminal_post_tool_call( @@ -1576,6 +1819,13 @@ def _execute(next_args: dict) -> Any: agent.interrupt("keyboard interrupt") except Exception: pass + # Emit a tool result for THIS call and every remaining call in + # the batch before re-raising (see interactive branch above). + _append_cancelled_tool_results( + messages, + assistant_message.tool_calls[i - 1:], + reason="keyboard interrupt", + ) raise except Exception as tool_error: function_result = f"Error executing tool '{function_name}': {tool_error}" @@ -1644,31 +1894,16 @@ def _execute(next_args: dict) -> Any: except Exception as _ver_err: logging.debug("file-mutation verifier record failed: %s", _ver_err) - if not _execution_blocked and agent.tool_progress_callback: - try: - agent.tool_progress_callback( - "tool.completed", function_name, None, None, - duration=tool_duration, is_error=_is_error_result, - result=function_result, - ) - except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") - agent._current_tool = None - agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s)") + _status_suffix = " (error)" if _is_error_result else "" + agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}") if agent.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") + logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) _log_result = _multimodal_text_summary(function_result) - logging.debug(f"Tool result ({len(_log_result)} chars): {_log_result}") - - if not _execution_blocked and agent.tool_complete_callback: - try: - display_args = _redact_tool_args_for_display(function_name, function_args) or function_args - agent.tool_complete_callback(tool_call.id, function_name, display_args, function_result) - except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") + logging.debug("Tool result (%d chars): %s", len(_log_result), _log_result) + display_function_result = function_result function_result = maybe_persist_tool_result( content=function_result, tool_name=function_name, @@ -1691,6 +1926,40 @@ def _execute(next_args: dict) -> Any: tool_message = make_tool_result_message(function_name, _tool_content, tool_call.id) messages.append(tool_message) risk_metadata = tool_message.get("_tool_output_risk") + if not _flush_session_db_after_tool_progress( + agent, + messages, + stage=f"tool result {function_name}", + ): + return + + # UI completion/progress events are projections of the canonical tool + # row, never a competing in-memory authority. + if not _execution_blocked and agent.tool_progress_callback: + try: + agent.tool_progress_callback( + "tool.completed", function_name, None, None, + duration=tool_duration, is_error=_is_error_result, + result=display_function_result, + ) + except Exception as cb_err: + logging.debug("Tool progress callback error: %s", cb_err) + + if not _execution_blocked and agent.tool_complete_callback: + try: + display_args = ( + _redact_tool_args_for_display(function_name, function_args) + or function_args + ) + agent.tool_complete_callback( + tool_call.id, + function_name, + display_args, + display_function_result, + ) + except Exception as cb_err: + logging.debug("Tool complete callback error: %s", cb_err) + if ( risk_metadata is not None and risk_metadata.get("risk") != "low" @@ -1707,17 +1976,6 @@ def _execute(next_args: dict) -> Any: ) except Exception as cb_err: logging.debug("Tool output risk callback error: %s", cb_err) - _flush_session_db_after_tool_progress( - agent, - messages, - stage=f"tool result {function_name}", - ) - - # ── Per-tool /steer drain ─────────────────────────────────── - # Drain pending steer BETWEEN individual tool calls so the - # injection lands as soon as a tool finishes — not after the - # entire batch. The model sees it on the next API iteration. - agent._apply_pending_steer_to_tool_results(messages, 1) if not agent.quiet_mode and getattr(agent, "tool_progress_mode", "all") != "off": if agent.verbose_logging: @@ -1739,17 +1997,18 @@ def _execute(next_args: dict) -> Any: skipped_tc.id, effect_disposition="none", )) - _flush_session_db_after_tool_progress( + if not _flush_session_db_after_tool_progress( agent, messages, stage=f"skipped tool result {skipped_name}", - ) + ): + return break - if agent.tool_delay > 0 and i < len(assistant_message.tool_calls): - time.sleep(agent.tool_delay) - # ── Per-turn aggregate budget enforcement ───────────────────────── + # Keep /steer pending until the final post-budget drain below. The model + # only receives this batch after all calls finish, and an early drain can + # be discarded when aggregate budget enforcement replaces a tool result. num_tools_seq = len(assistant_message.tool_calls) if finalize and num_tools_seq > 0: enforce_turn_budget(messages[-num_tools_seq:], env=get_active_env(effective_task_id), config=_tool_budget) @@ -1794,6 +2053,8 @@ def execute_tool_calls_segmented(agent, assistant_message, messages: list, effec segments = _plan_tool_batch_segments(assistant_message.tool_calls, execution_cwd=_exec_cwd) for kind, calls in segments: + if getattr(agent, "_incremental_persistence_failed", False): + return segment_message = SimpleNamespace(tool_calls=list(calls)) if kind == "parallel": execute_tool_calls_concurrent( @@ -1806,6 +2067,9 @@ def execute_tool_calls_segmented(agent, assistant_message, messages: list, effec finalize=False, ) + if getattr(agent, "_incremental_persistence_failed", False): + return + # ── Whole-turn finalize (budget + /steer) ───────────────────────── total_tools = len(assistant_message.tool_calls) if total_tools > 0: diff --git a/agent/tool_guardrails.py b/agent/tool_guardrails.py index f08f1b604786..444ce3739596 100644 --- a/agent/tool_guardrails.py +++ b/agent/tool_guardrails.py @@ -79,6 +79,7 @@ class ToolCallGuardrailConfig: no_progress_block_after: int = 5 idempotent_tools: frozenset[str] = field(default_factory=lambda: IDEMPOTENT_TOOL_NAMES) mutating_tools: frozenset[str] = field(default_factory=lambda: MUTATING_TOOL_NAMES) + loop_caps: "LoopCapConfig" = field(default_factory=lambda: LoopCapConfig()) @classmethod def from_mapping(cls, data: Mapping[str, Any] | None) -> "ToolCallGuardrailConfig": @@ -121,6 +122,54 @@ def from_mapping(cls, data: Mapping[str, Any] | None) -> "ToolCallGuardrailConfi hard_stop_after.get("idempotent_no_progress", data.get("no_progress_block_after")), defaults.no_progress_block_after, ), + loop_caps=LoopCapConfig.from_mapping(data.get("loop_caps")), + ) + + +# Default session-wide caps, matching Claude Code's v2.1.212 runaway-loop +# Per-turn (per-agent-loop) caps on runaway-prone tool calls. Counts reset at +# the start of every agent loop (reset_for_turn), so the limit is "within a +# single turn" rather than cumulative over the whole session. A single loop +# issuing dozens of web searches or spawning dozens of subagents is already +# pathological, so the defaults are deliberately low. +_DEFAULT_MAX_WEB_SEARCHES_PER_TURN = 50 +_DEFAULT_MAX_SUBAGENTS_PER_TURN = 50 + + +@dataclass(frozen=True) +class LoopCapConfig: + """Per-turn caps on runaway-prone tool calls. + + Inspired by Claude Code v2.1.212 (Week 29, July 2026), which added caps on + WebSearch calls and subagent spawns to stop runaway search / delegation + loops. Here the caps count *within a single agent loop* (one turn): the + counters reset in ``reset_for_turn`` at the start of every + ``run_conversation``, so a legitimate multi-turn session is never starved, + but a single turn that spirals into an unbounded search / delegation loop + is stopped. + + Semantics differ from the per-turn loop *detector* above (which keys on + repeated identical/failing calls): these caps are a hard ceiling on the + total count of a tool within the turn and fire regardless of + ``hard_stop_enabled``. A value of ``0`` disables the cap (unlimited). + """ + + max_web_searches: int = _DEFAULT_MAX_WEB_SEARCHES_PER_TURN + max_subagents: int = _DEFAULT_MAX_SUBAGENTS_PER_TURN + + @classmethod + def from_mapping(cls, data: Mapping[str, Any] | None) -> "LoopCapConfig": + """Build config from the ``tool_loop_guardrails.loop_caps`` section.""" + if not isinstance(data, Mapping): + return cls() + defaults = cls() + return cls( + max_web_searches=_non_negative_int( + data.get("max_web_searches"), defaults.max_web_searches + ), + max_subagents=_non_negative_int( + data.get("max_subagents"), defaults.max_subagents + ), ) @@ -233,6 +282,11 @@ def reset_for_turn(self) -> None: self._same_tool_failure_counts: dict[str, int] = {} self._no_progress: dict[ToolCallSignature, tuple[str, int]] = {} self._halt_decision: ToolGuardrailDecision | None = None + # Per-turn runaway-loop cap counters. Reset every turn (this method + # runs at the start of each run_conversation), so the caps bound a + # single agent loop rather than accumulating across the session. + self._turn_web_search_count = 0 + self._turn_subagent_count = 0 @property def halt_decision(self) -> ToolGuardrailDecision | None: @@ -240,6 +294,17 @@ def halt_decision(self) -> ToolGuardrailDecision | None: def before_call(self, tool_name: str, args: Mapping[str, Any] | None) -> ToolGuardrailDecision: signature = ToolCallSignature.from_call(tool_name, _coerce_args(args)) + + # ── Per-turn runaway-loop caps ────────────────────────────────── + # These are hard ceilings on how many times a runaway-prone tool may + # be called within a single agent loop (turn). They apply regardless + # of hard_stop_enabled (which only governs the per-turn loop detector). + # We block BEFORE the call runs once the count is already at the cap, + # then increment for an allowed call so the (cap+1)-th is refused. + cap_block = self._check_loop_cap(tool_name, _coerce_args(args), signature) + if cap_block is not None: + return cap_block + if not self.config.hard_stop_enabled: return ToolGuardrailDecision(tool_name=tool_name, signature=signature) @@ -379,6 +444,68 @@ def _is_idempotent(self, tool_name: str) -> bool: return False return tool_name in self.config.idempotent_tools + def _check_loop_cap( + self, + tool_name: str, + args: Mapping[str, Any], + signature: ToolCallSignature, + ) -> ToolGuardrailDecision | None: + """Enforce and advance the per-turn runaway-loop counters. + + Returns a ``block`` decision when the cap is already reached, otherwise + increments the relevant counter for the allowed call and returns + ``None``. A cap of 0 disables that limit entirely. Counters reset each + turn via ``reset_for_turn``. + """ + caps = self.config.loop_caps + + if tool_name == "web_search": + cap = caps.max_web_searches + if cap and self._turn_web_search_count >= cap: + decision = ToolGuardrailDecision( + action="block", + code="loop_web_search_cap", + message=( + f"Blocked web_search: this turn has already made {cap} " + "web searches, the per-turn limit. This looks like a " + "runaway search loop. Work with the results you already " + "have and give the user your answer." + ), + tool_name=tool_name, + count=self._turn_web_search_count, + signature=signature, + ) + self._halt_decision = decision + return decision + self._turn_web_search_count += 1 + return None + + if tool_name == "delegate_task": + cap = caps.max_subagents + if not cap: + return None + spawn_count = _subagent_spawn_count(args) + if self._turn_subagent_count >= cap: + decision = ToolGuardrailDecision( + action="block", + code="loop_subagent_cap", + message=( + f"Blocked delegate_task: this turn has already spawned " + f"{self._turn_subagent_count} subagents (limit {cap}). " + "This looks like a runaway delegation loop. Finish the " + "work with the results you have and answer the user." + ), + tool_name=tool_name, + count=self._turn_subagent_count, + signature=signature, + ) + self._halt_decision = decision + return decision + self._turn_subagent_count += spawn_count + return None + + return None + def toolguard_synthetic_result(decision: ToolGuardrailDecision) -> str: """Build a synthetic role=tool content string for a blocked tool call.""" @@ -471,6 +598,32 @@ def _positive_int(value: Any, default: int) -> int: return parsed if parsed >= 1 else default +def _non_negative_int(value: Any, default: int) -> int: + """Parse a session-cap value. 0 is a valid (disable) value; negatives and + junk fall back to the default.""" + if value is None: + return default + try: + parsed = int(value) + except (TypeError, ValueError): + return default + return parsed if parsed >= 0 else default + + +def _subagent_spawn_count(args: Mapping[str, Any]) -> int: + """How many subagents a single delegate_task call spawns. + + delegate_task runs in one of two modes: a batch (``tasks`` is a non-empty + list, one child per item) or a single task (``goal``). Count the batch size + when present, otherwise 1, so the session subagent cap reflects real spawns + rather than delegate_task invocations. + """ + tasks = args.get("tasks") if isinstance(args, Mapping) else None + if isinstance(tasks, list) and tasks: + return len(tasks) + return 1 + + def _sha256(value: str) -> str: # surrogatepass: tool results scraped from the web can carry unpaired # UTF-16 surrogates (e.g. half of a mathematical-bold pair); a strict diff --git a/agent/trace_upload.py b/agent/trace_upload.py index f65547440c7f..404d9be70b13 100644 --- a/agent/trace_upload.py +++ b/agent/trace_upload.py @@ -162,7 +162,7 @@ def build_trace_jsonl( if cwd: r = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], - capture_output=True, text=True, timeout=3, cwd=cwd, + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3, cwd=cwd, ) if r.returncode == 0: git_branch = r.stdout.strip() diff --git a/agent/transports/chat_completions.py b/agent/transports/chat_completions.py index 086883eca1c7..2572038126b6 100644 --- a/agent/transports/chat_completions.py +++ b/agent/transports/chat_completions.py @@ -9,6 +9,7 @@ reasoning configuration, temperature handling, and extra_body assembly. """ +import json from typing import Any, Dict from agent.lmstudio_reasoning import resolve_lmstudio_effort @@ -18,6 +19,56 @@ from agent.transports.types import NormalizedResponse, ToolCall, Usage +def _static_prompt_instructions(messages: list[dict[str, Any]]) -> str: + """Return the stable system/developer prefix used for cache routing. + + Chat Completions carries instructions in its message list rather than a + separate ``instructions`` field. Only a leading system/developer message + is static by contract; later messages are conversation state and must not + split a warm prefix bucket on every turn. + """ + if not messages or not isinstance(messages[0], dict): + return "" + first = messages[0] + if first.get("role") not in {"system", "developer"}: + return "" + content = first.get("content") + if isinstance(content, str): + return content + try: + return json.dumps(content, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + except (TypeError, ValueError): + return str(content or "") + + +def _add_prompt_cache_key( + api_kwargs: dict[str, Any], + *, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None, + supports_prompt_cache_key: bool, +) -> None: + """Add a content-addressed key only for an explicitly capable endpoint.""" + if not supports_prompt_cache_key: + return + + # An explicit caller body field is authoritative too. Do not add a + # duplicate top-level field whose SDK merge precedence could overwrite it. + extra_body = api_kwargs.get("extra_body") + if "prompt_cache_key" in api_kwargs or ( + isinstance(extra_body, dict) and "prompt_cache_key" in extra_body + ): + return + + # Reuse the Responses transport's single authoritative hash algorithm so + # equivalent static prefixes route to the same cache bucket across modes. + from agent.transports.codex import _content_cache_key + + cache_key = _content_cache_key(_static_prompt_instructions(messages), tools) + if cache_key: + api_kwargs["prompt_cache_key"] = cache_key + + def _reasoning_config_for_model(model: str, reasoning_config: dict | None) -> dict | None: """Return the model's wire-compatible reasoning config.""" if not isinstance(reasoning_config, dict): @@ -112,6 +163,24 @@ def _is_gemini_openai_compat_base_url(base_url: Any) -> bool: return normalized.endswith("/openai") +def _is_openai_api_base_url(base_url: Any) -> bool: + """True only for api.openai.com itself (exact host). + + OpenAI documents ``prompt_cache_key`` as a first-class body field and + GPT-5.6+ docs recommend it for reliable cache routing, so the flag is + implied for the real endpoint. Deliberately NOT a substring match: + Azure OpenAI and strict OpenAI-compat endpoints may reject unknown + fields and must stay opt-in via ``supports_prompt_cache_key``. + """ + try: + from urllib.parse import urlparse + + host = (urlparse(str(base_url or "").strip()).hostname or "").lower() + except Exception: + return False + return host == "api.openai.com" + + def _model_consumes_thought_signature(model: Any) -> bool: """True when the outgoing model is a Gemini family model that requires ``extra_content`` (thought_signature) to be replayed on tool calls. @@ -327,6 +396,8 @@ def build_kwargs( # Claude on OpenRouter/Nous max output anthropic_max_output: int | None extra_body_additions: dict | None + supports_prompt_cache_key: bool — explicit endpoint capability for + the top-level Chat Completions request field; defaults off. """ # Codex sanitization: drop reasoning_items / call_id / response_item_id. # Pass model so the Gemini thought_signature (extra_content) is kept for @@ -378,7 +449,6 @@ def build_kwargs( ephemeral = params.get("ephemeral_max_output_tokens") max_tokens = params.get("max_tokens") anthropic_max_out = params.get("anthropic_max_output") - is_nvidia_nim = params.get("is_nvidia_nim", False) is_kimi = params.get("is_kimi", False) is_tokenhub = params.get("is_tokenhub", False) reasoning_config = _reasoning_config_for_model(model, params.get("reasoning_config")) @@ -436,7 +506,6 @@ def build_kwargs( extra_body: dict[str, Any] = {} is_openrouter = params.get("is_openrouter", False) - is_nous = params.get("is_nous", False) is_github_models = params.get("is_github_models", False) provider_name = str(params.get("provider_name") or "").strip().lower() base_url = params.get("base_url") @@ -509,6 +578,14 @@ def build_kwargs( if overrides: api_kwargs.update(overrides) + _add_prompt_cache_key( + api_kwargs, + messages=sanitized, + tools=api_kwargs.get("tools"), + supports_prompt_cache_key=bool(params.get("supports_prompt_cache_key")) + or _is_openai_api_base_url(params.get("base_url")), + ) + return api_kwargs def _build_kwargs_from_profile(self, profile, model, sanitized, tools, params): @@ -651,6 +728,13 @@ def _build_kwargs_from_profile(self, profile, model, sanitized, tools, params): if extra_body: api_kwargs["extra_body"] = extra_body + _add_prompt_cache_key( + api_kwargs, + messages=sanitized, + tools=api_kwargs.get("tools"), + supports_prompt_cache_key=bool(profile.supports_prompt_cache_key), + ) + return api_kwargs def normalize_response(self, response: Any, **kwargs) -> NormalizedResponse: diff --git a/agent/transports/codex.py b/agent/transports/codex.py index 5855fcfe9c54..de71c2b50006 100644 --- a/agent/transports/codex.py +++ b/agent/transports/codex.py @@ -7,6 +7,7 @@ import hashlib import json +import re from typing import Any, Dict, List, Optional from agent.transports.base import ProviderTransport @@ -27,6 +28,49 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]: return f"pck_{digest}" +_EXTENDED_PROMPT_CACHE_MODELS = ( + "gpt-5.5-pro", + "gpt-5.5", + "gpt-5.4", + "gpt-5.2", + "gpt-5.1-codex-max", + "gpt-5.1-codex-mini", + "gpt-5.1-chat-latest", + "gpt-5.1-codex", + "gpt-5.1", + "gpt-5-codex", + "gpt-5", + "gpt-4.1", +) +_EXTENDED_PROMPT_CACHE_MODEL_RE = re.compile( + rf"(?:^|[./:])(?:{'|'.join(re.escape(name) for name in _EXTENDED_PROMPT_CACHE_MODELS)})" + r"(?:-\d{4}-\d{2}-\d{2})?$" +) + + +def _default_prompt_cache_retention_for_request( + model: str, + base_url: Any, +) -> Optional[str]: + """Return ``24h`` for supported models on Amazon Bedrock Mantle.""" + from utils import base_url_hostname + + hostname_parts = base_url_hostname(str(base_url or "")).split(".") + is_bedrock_mantle = ( + len(hostname_parts) == 4 + and hostname_parts[0] == "bedrock-mantle" + and bool(hostname_parts[1]) + and hostname_parts[2:] == ["api", "aws"] + ) + if not is_bedrock_mantle: + return None + + normalized = str(model or "").strip().lower().replace("_", "-") + if _EXTENDED_PROMPT_CACHE_MODEL_RE.search(normalized): + return "24h" + return None + + def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]: """Content-address the prompt cache key from the static request prefix. @@ -284,6 +328,13 @@ def build_kwargs( if not is_github_responses and not is_xai_responses and cache_key: kwargs["prompt_cache_key"] = cache_key + cache_retention = _default_prompt_cache_retention_for_request( + model, + params.get("base_url"), + ) + if cache_retention: + kwargs.setdefault("prompt_cache_retention", cache_retention) + if reasoning_enabled and is_xai_responses: from agent.model_metadata import grok_supports_reasoning_effort @@ -493,10 +544,13 @@ def preflight_kwargs( *, allow_stream: bool = False, is_github_responses: bool = False, + sanitize_harmony_tokens: bool = False, ) -> dict: """Validate and sanitize Codex API kwargs before the call. Normalizes input items, strips unsupported fields, validates structure. + ``sanitize_harmony_tokens`` is enabled only for the ChatGPT Codex + backend, which rejects literal reserved Harmony wire tokens in text. """ from agent.codex_responses_adapter import _preflight_codex_api_kwargs @@ -504,6 +558,7 @@ def preflight_kwargs( api_kwargs, allow_stream=allow_stream, is_github_responses=is_github_responses, + sanitize_harmony_tokens=sanitize_harmony_tokens, ) if "prompt_cache_key" in normalized: bounded = _bounded_prompt_cache_key(normalized["prompt_cache_key"]) diff --git a/agent/transports/codex_app_server.py b/agent/transports/codex_app_server.py index 7f5831f2a3ea..c23ff836ed89 100644 --- a/agent/transports/codex_app_server.py +++ b/agent/transports/codex_app_server.py @@ -394,7 +394,7 @@ def check_codex_binary( proc = subprocess.run( [codex_bin, "--version"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=10, stdin=subprocess.DEVNULL, ) diff --git a/agent/transports/codex_app_server_session.py b/agent/transports/codex_app_server_session.py index 7954ecbf4d52..384a7de8ab81 100644 --- a/agent/transports/codex_app_server_session.py +++ b/agent/transports/codex_app_server_session.py @@ -1054,6 +1054,18 @@ def _handle_server_request(self, req: dict) -> None: ) def _decide_exec_approval(self, params: dict) -> str: + """Decide a Codex exec approval request. + + This is protocol-level routing only — it carries NO Hermes + approval-mode/timeout logic. The Hermes-side resolution happens + upstream: ``agent/codex_runtime.py`` derives + ``auto_approve_exec`` from the canonical + ``tools.approval.is_approval_bypass_active()`` (which reads + ``approvals.mode`` via ``tools.approval._get_approval_mode``), + and ``self._approval_callback`` itself runs the shared approval + gate (mode + ``approvals.timeout``) in ``tools/approval.py``. + Keep it that way — do not re-read approval config here. + """ if self._routing.auto_approve_exec: return "accept" command = params.get("command") or "" @@ -1077,6 +1089,12 @@ def _decide_exec_approval(self, params: dict) -> str: return "decline" # fail-closed when no callback wired def _decide_apply_patch_approval(self, params: dict) -> str: + """Decide a Codex apply_patch approval request. + + Protocol-level routing only; Hermes approval-mode/timeout + resolution is delegated to ``tools/approval.py`` upstream — see + the docstring on ``_decide_exec_approval``. + """ if self._routing.auto_approve_apply_patch: return "accept" if self._approval_callback is not None: @@ -1231,11 +1249,18 @@ def _approval_choice_to_codex_decision(choice: str) -> str: Codex expects 'accept', 'acceptForSession', 'decline', or 'cancel' (verified against codex-rs/app-server-protocol/src/protocol/v2/item.rs on codex 0.130.0). + + This mapping is Codex-protocol-semantic and intentionally lives here, + NOT in tools/approval.py: the Hermes approval mode/timeout resolution + and the choice itself come from the shared core (tools/approval.py); + only the wire-value translation is local. """ if choice in {"once",}: return "accept" if choice in {"session", "always"}: return "acceptForSession" + # "deny" and "timeout" both map to decline — codex has no wire value for + # "prompt expired"; the Hermes-side messaging already distinguishes them. return "decline" diff --git a/agent/turn_context.py b/agent/turn_context.py index 883caa204f98..def497b15898 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -36,10 +36,12 @@ PREFLIGHT_COMPRESSION_STATUS_TEMPLATE, compression_skipped_due_to_lock, conversation_history_after_compression, + recover_rotated_compression_session, ) from agent.context_engine import automatic_compaction_status_message from agent.iteration_budget import IterationBudget from agent.memory_manager import build_memory_context_block +from agent.memory_provider import is_trivial_prompt from agent.model_metadata import ( estimate_messages_tokens_rough, estimate_request_tokens_rough, @@ -335,6 +337,8 @@ def build_turn_context( persist_user_message: Optional[Any], persist_user_timestamp: Optional[float] = None, *, + persist_user_display_kind: Optional[str] = None, + persist_user_display_metadata: Optional[Dict[str, Any]] = None, restore_or_build_system_prompt, install_safe_stdio, sanitize_surrogates, @@ -353,6 +357,13 @@ def build_turn_context( # Guard stdio against OSError from broken pipes (systemd/headless/daemon). install_safe_stdio() + # Recover a session rotated by another path before binding log/turn ids or + # copying client-supplied history. Everything in this turn must consistently + # belong to the canonical child, including observability metadata. + recovered_history = recover_rotated_compression_session(agent) + if recovered_history is not None: + conversation_history = recovered_history + # NOTE: the DB session row is created later, AFTER the system prompt is # restored/built (see _ensure_db_session() below the system-prompt block). # Creating it here — before _cached_system_prompt is populated — inserts a @@ -428,7 +439,12 @@ def build_turn_context( # Generate unique task_id if not provided to isolate VMs between tasks. effective_task_id = task_id or str(uuid.uuid4()) agent._current_task_id = effective_task_id - turn_id = f"{agent.session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + turn_id = str(getattr(agent, "_relay_pending_turn_id", "") or "") + if not turn_id: + turn_id = ( + f"{agent.session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" + ) + agent._relay_pending_turn_id = None agent._current_turn_id = turn_id agent._current_api_request_id = "" # Tripwire: warn (with both turn ids) when this turn starts before the @@ -529,6 +545,19 @@ def build_turn_context( # Add the current user message after the prompt/session setup has made # close persistence safe. The handoff above preserves any marker already # stamped by an earlier close flush. + # + # A synthesized turn (auto-continue recovery note, delegation completion) + # declares how it should READ in a transcript. Stamp that on the live + # message so the crash persist below writes the row already typed. Typing + # it after the turn instead leaves the row untyped for the whole run — and + # forever if the turn crashes — so the raw system note paints as a user + # bubble. The model still receives role/content unchanged; the api_messages + # build strips both fields from every outgoing copy. + if persist_user_display_kind: + user_msg["display_kind"] = persist_user_display_kind + if persist_user_display_metadata: + user_msg["display_metadata"] = persist_user_display_metadata + messages.append(user_msg) current_turn_user_idx = len(messages) - 1 agent._persist_user_message_idx = current_turn_user_idx @@ -642,17 +671,9 @@ def build_turn_context( ) # Post-compression target size: don't summarise a thread already # below what compaction would reduce it to. - _configured_idle_ratio = getattr( - agent, "compression_summary_target_ratio", 0.20 + _idle_floor = int( + _compressor.threshold_tokens * _compressor.summary_target_ratio ) - _idle_ratio = getattr( - _compressor, "summary_target_ratio", _configured_idle_ratio - ) - if isinstance(_idle_ratio, bool) or not isinstance( - _idle_ratio, (int, float) - ): - _idle_ratio = _configured_idle_ratio - _idle_floor = int(_compressor.threshold_tokens * _idle_ratio) _idle_cooldown = getattr( _compressor, "get_active_compression_failure_cooldown", lambda: None )() @@ -1030,7 +1051,7 @@ def build_turn_context( # Plugin hook: pre_llm_call (context injected into user message, not system prompt). plugin_user_context = "" try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _pre_results = _invoke_hook( "pre_llm_call", session_id=agent.session_id, @@ -1041,6 +1062,7 @@ def build_turn_context( is_first_turn=(not bool(conversation_history)), model=agent.model, platform=getattr(agent, "platform", None) or "", + parent_session_id=getattr(agent, "_parent_session_id", None) or "", sender_id=getattr(agent, "_user_id", None) or "", ) _ctx_parts: list[str] = [] @@ -1131,11 +1153,15 @@ def build_turn_context( pass # External memory provider: prefetch once before the tool loop. + # + # Skip prefetch on trivial prompts (greetings, acknowledgements) to + # prevent memory-context injection on turns that carry no semantic signal. ext_prefetch_cache = "" if agent._memory_manager: try: _query = original_user_message if isinstance(original_user_message, str) else "" - ext_prefetch_cache = agent._memory_manager.prefetch_all(_query) or "" + if not is_trivial_prompt(_query): + ext_prefetch_cache = agent._memory_manager.prefetch_all(_query) or "" except Exception: pass diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 4e2d318b2e2c..d4d6a23e7865 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -339,6 +339,12 @@ def finalize_turn( # otherwise ``/resume`` reloads ``content=""`` and the bug # resurfaces cross-session. _tail.pop("_db_persisted", None) + # The bounded flush-scan cursor (run_agent.py) skips the + # identity-matched prefix of its previous snapshot on the + # assumption that no live dict loses the marker in place — + # this pop is the one place that does. Invalidate it so the + # filled row is re-examined instead of skipped. + agent._db_flush_scan_prefix = None # The model has completed its request, so replace API-local # voice/model/skill guidance with the clean user input before writing the @@ -349,6 +355,58 @@ def finalize_turn( _apply_override = getattr(agent, "_apply_persist_user_message_override", None) if callable(_apply_override): _apply_override(messages) + + # ── Post-turn micro-compaction ──────────────────────────── + # After the assistant response is finalized but before the session is + # persisted, run micro-compaction to absorb the oldest uncompacted + # exchange into the rolling summary. This amortizes compression + # across turns rather than batching it into one big pause. + if not interrupted and not failed: + try: + _compressor = getattr(agent, "context_compressor", None) + # Strict `is True` + isinstance gates: plugin context engines + # (and MagicMock compressors in tests) satisfy getattr/duck + # checks with truthy auto-attributes — a bare truthiness check + # here called _micro_compact on a mock and spliced its (empty- + # iterating) return value over the transcript, wiping it. + if ( + _compressor + and getattr(_compressor, '_micro_compact_enabled', False) is True + and callable(getattr(_compressor, '_micro_compact', None)) + and final_response + # Persistence-isolated agents (background review fork) + # must not micro-compact: the pass burns a real aux-LLM + # call on a throwaway replay transcript, and if the + # compressor ever holds a session_db binding it would + # archive_and_compact the CANONICAL session rows — the + # exact write class _persist_disabled exists to stop. + and not getattr(agent, "_persist_disabled", False) + ): + _before = len(messages) + _compacted = _compressor._micro_compact(messages) + # Micro-compaction defrag rewrites the newest MICRO + # marker's content and pops _db_persisted from the live + # dict in place — the sibling of the pop site above. The + # compressor has no agent reference, so it raises a flag + # for us to invalidate the bounded flush-scan cursor; + # otherwise the rewritten marker row is identity-skipped + # and the stale summary persists to state.db. + if getattr( + _compressor, "_flush_scan_cursor_invalidated", False + ): + _compressor._flush_scan_cursor_invalidated = False + agent._db_flush_scan_prefix = None + if isinstance(_compacted, list) and _compacted: + messages[:] = _compacted + _after = len(messages) + if _before != _after: + logger.info( + "Micro-compaction: %d -> %d messages", + _before, _after, + ) + except Exception as _mc_err: + logger.info("Micro-compaction failed: %s", _mc_err) + agent._persist_session(messages, conversation_history) except Exception as _persist_err: _cleanup_errors.append(f"persist_session: {_persist_err}") @@ -488,7 +546,7 @@ def finalize_turn( # First hook to return a string wins; None/empty return leaves text unchanged. if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _transform_results = _invoke_hook( "transform_llm_output", response_text=final_response, @@ -510,7 +568,7 @@ def finalize_turn( # to an external memory system). if final_response and not interrupted: try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "post_llm_call", session_id=agent.session_id, @@ -607,6 +665,13 @@ def finalize_turn( } if agent._tool_guardrail_halt_decision is not None: result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata() + # Persistence failures already set failed=True + an explanation in + # final_response; also stamp `error` so gateway surfaces status="error" + # (and desktop can toast disk-full) instead of a quiet complete frame. + if failed and str(_turn_exit_reason) == "session_persistence_failed": + result["error"] = final_response or ( + "session storage could not be written — free disk space and try again" + ) # Surface any post-loop cleanup failures so the caller can distinguish a # clean turn from one whose trajectory/session/resource teardown raised # (the response is still returned either way — #8049). @@ -669,14 +734,16 @@ def finalize_turn( # Fired at the very end of every run_conversation call. # Plugins can use this for cleanup, flushing buffers, etc. try: - from hermes_cli.plugins import invoke_hook as _invoke_hook + from hermes_cli.lifecycle import invoke_hook as _invoke_hook _invoke_hook( "on_session_end", session_id=agent.session_id, task_id=effective_task_id, turn_id=turn_id, completed=completed, + failed=failed, interrupted=interrupted, + turn_exit_reason=_turn_exit_reason, model=agent.model, platform=getattr(agent, "platform", None) or "", ) diff --git a/agent/turn_retry_state.py b/agent/turn_retry_state.py index 59e343bfeda3..d73fe5b6bfc8 100644 --- a/agent/turn_retry_state.py +++ b/agent/turn_retry_state.py @@ -45,6 +45,14 @@ class TurnRetryState: nous_auth_retry_attempted: bool = False nous_paid_entitlement_refresh_attempted: bool = False copilot_auth_retry_attempted: bool = False + # Copilot surfaces a stale/degraded credential as a 400 + # ``model_not_available_for_integrator`` / ``model_not_supported`` instead + # of a clean 401 (e.g. a raw OAuth token seeded when the token exchange + # degraded at startup, routing the request to the restricted + # ``copilot-language-server`` integrator). Guard a single-shot forced + # re-exchange + client rebuild for that case, separate from the 401 guard + # so both can fire within one attempt if needed. + copilot_stale_cred_retry_attempted: bool = False vertex_auth_retry_attempted: bool = False # ── Format / payload recovery guards ───────────────────────────────── diff --git a/agent/turn_summary.py b/agent/turn_summary.py new file mode 100644 index 000000000000..f4440afb50aa --- /dev/null +++ b/agent/turn_summary.py @@ -0,0 +1,310 @@ +"""Per-turn accounting for the interactive CLI. + +Two display-only pieces live here: + +* :class:`TurnSummaryCollector` — a tiny observer that rides the existing + ``tool_progress_callback`` feed (``tool.completed`` events already carry + the tool name and its raw result) and tallies what a turn actually did. + It holds **no** agent-loop state: the display layer already sees every + tool call, so nothing new is threaded through the conversation loop. +* :func:`format_turn_summary` — a pure formatter that turns a tally plus a + wall-clock duration into one dim line, e.g.:: + + ⋯ 12.4s · edited 2 files +18 -3 · read 4 files · ran 3 commands + + Ported from Claude Code's post-turn accounting line + ("Edited 1 file +6 -2, read 1 file … Worked for 10s"). + +:func:`format_token_flow` is the spinner-side counterpart: a cumulative +token readout appended to the live elapsed timer (``↓ 1.2k tok``). + +Everything in this module is pure/side-effect free apart from the +collector's own counters, which makes it directly unit-testable without a +terminal, an agent, or a network call. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +__all__ = [ + "TurnSummaryCollector", + "TurnTally", + "format_turn_summary", + "format_token_flow", + "format_elapsed", +] + + +# Leading glyph for the summary line. Deliberately not an emoji — the line is +# meant to read as terminal chrome, not as agent speech. +SUMMARY_PREFIX = "⋯" + +# A turn that called no tools and finished this fast has nothing worth +# reporting (plain chat reply). Below the threshold the formatter returns "". +_MIN_TOOLLESS_SECONDS = 2.0 + +# Max number of "verb + count" segments rendered before collapsing the rest +# into a "+N more" tail, so a 12-tool turn cannot blow past one line. +_MAX_SEGMENTS = 4 + + +# Tool name -> (verb, singular noun, plural noun). +# +# Verbs are past tense because the line is printed *after* the turn. Tools not +# listed here fall into a generic "called N tools" bucket rather than inventing +# phrasing for plugin/MCP tools whose semantics we don't know. +_VERB_GROUPS: dict[str, tuple[str, str, str]] = { + "write_file": ("edited", "file", "files"), + "patch": ("edited", "file", "files"), + "read_file": ("read", "file", "files"), + "web_extract": ("read", "page", "pages"), + "terminal": ("ran", "command", "commands"), + "execute_code": ("ran", "script", "scripts"), + "search_files": ("searched", "path", "paths"), + "web_search": ("searched the web", "time", "times"), + "session_search": ("searched sessions", "time", "times"), + "browser_navigate": ("browsed", "page", "pages"), + "skill_view": ("read", "skill", "skills"), + "skill_manage": ("updated", "skill", "skills"), + "skills_list": ("listed skills", "time", "times"), + "todo": ("updated", "task list", "task lists"), + "delegate_task": ("delegated", "task", "tasks"), + "memory": ("updated", "memory", "memories"), +} + +# Verb groups that carry file-edit line deltas (+X -Y) when known. +_EDIT_VERB = "edited" + +# Render order: edits first (the thing users most want confirmed), then reads, +# then commands. Anything else follows in first-seen order. +_VERB_PRIORITY: tuple[str, ...] = ("edited", "read", "ran") + +# Tools whose results may report a unified diff we can count lines from. +_DIFF_RESULT_TOOLS = frozenset({"patch"}) + + +@dataclass +class TurnTally: + """What a single turn did, as observed from the tool-progress feed.""" + + # verb -> {noun_plural: count}; keeps insertion order for stable rendering. + verbs: dict[str, dict[str, int]] = field(default_factory=dict) + # Tools with no curated verb, counted together. + other_tools: int = 0 + # Aggregated unified-diff line deltas across edit tools, when reported. + lines_added: int = 0 + lines_removed: int = 0 + # True once at least one edit tool reported a countable diff, so the + # formatter knows the difference between "+0 -0" and "unknown". + has_line_deltas: bool = False + + @property + def total_tools(self) -> int: + counted = sum(sum(nouns.values()) for nouns in self.verbs.values()) + return counted + self.other_tools + + +def _count_diff_lines(diff: str) -> tuple[int, int]: + """Count added/removed lines in unified-diff text. + + File headers (``+++``/``---``) are excluded so a one-line edit does not + read as three additions. + """ + added = removed = 0 + for line in diff.splitlines(): + if line.startswith("+++") or line.startswith("---"): + continue + if line.startswith("+"): + added += 1 + elif line.startswith("-"): + removed += 1 + return added, removed + + +def _extract_line_deltas(tool_name: str, result: Any) -> tuple[int, int] | None: + """Pull (added, removed) from a tool result, or None when unavailable. + + Only tools that already report a diff in their result payload are + inspected — we never shell out to git and never re-read files to + synthesise a delta. + """ + if tool_name not in _DIFF_RESULT_TOOLS: + return None + payload: Any = result + if isinstance(payload, str): + text = payload.strip() + if not text.startswith("{"): + return None + try: + import json + + # strict=False tolerates literal control characters inside strings + # (raw newlines in an embedded diff), which some tool serialisers + # emit. A tally line is never worth failing over formatting. + payload = json.loads(text, strict=False) + except Exception: + return None + if not isinstance(payload, dict): + return None + diff = payload.get("diff") + if not isinstance(diff, str) or not diff.strip(): + return None + added, removed = _count_diff_lines(diff) + # A diff that carries no +/- content lines (e.g. a bare hunk header) tells + # us nothing — report it as unknown rather than rendering a misleading + # "+0 -0" next to a real edit. + if added == 0 and removed == 0: + return None + return added, removed + + +class TurnSummaryCollector: + """Accumulate per-turn tool tallies from the tool-progress feed. + + Wired into the CLI's existing ``_on_tool_progress`` handler: the display + layer already receives every ``tool.completed`` event with the tool name + and raw result, so no agent-loop bookkeeping is added. + """ + + def __init__(self) -> None: + self._tally = TurnTally() + + def begin(self) -> None: + """Start a fresh turn (drops any prior tally).""" + self._tally = TurnTally() + + def record_tool( + self, + tool_name: str | None, + *, + result: Any = None, + is_error: bool = False, + ) -> None: + """Record one completed tool call. + + Failed calls are skipped: a summary claiming "edited 2 files" when one + write was denied would be exactly the over-claim the file-mutation + verifier exists to catch. + """ + if not tool_name or is_error: + return + # Internal/pseudo tools (``_thinking``) are not user-visible work. + if tool_name.startswith("_"): + return + + group = _VERB_GROUPS.get(tool_name) + if group is None: + self._tally.other_tools += 1 + return + + verb, _singular, plural = group + nouns = self._tally.verbs.setdefault(verb, {}) + nouns[plural] = nouns.get(plural, 0) + 1 + + if verb == _EDIT_VERB: + deltas = _extract_line_deltas(tool_name, result) + if deltas is not None: + added, removed = deltas + self._tally.lines_added += added + self._tally.lines_removed += removed + self._tally.has_line_deltas = True + + @property + def tally(self) -> TurnTally: + return self._tally + + def render(self, elapsed_seconds: float) -> str: + """Render this turn's summary line (see :func:`format_turn_summary`).""" + return format_turn_summary(elapsed_seconds, self._tally) + + +def format_elapsed(seconds: float) -> str: + """Format a wall-clock duration compactly (``12.4s`` / ``2m05s``).""" + if seconds < 0: + seconds = 0.0 + if seconds < 60: + return f"{seconds:.1f}s" + minutes, rest = divmod(int(round(seconds)), 60) + return f"{minutes}m{rest:02d}s" + + +def _pluralize(count: int, plural_noun: str) -> str: + """Return ``"1 file"`` / ``"3 files"`` from a plural noun form.""" + if count == 1: + singular = plural_noun + if plural_noun.endswith("ies"): + singular = plural_noun[:-3] + "y" + elif plural_noun.endswith("ses"): + singular = plural_noun[:-2] + elif plural_noun.endswith("s"): + singular = plural_noun[:-1] + return f"1 {singular}" + return f"{count} {plural_noun}" + + +def _ordered_verbs(tally: TurnTally) -> list[str]: + """Verbs in render order: priority verbs first, then first-seen order.""" + seen = list(tally.verbs.keys()) + ranked = [v for v in _VERB_PRIORITY if v in tally.verbs] + ranked += [v for v in seen if v not in _VERB_PRIORITY] + return ranked + + +def format_turn_summary( + elapsed_seconds: float, + tally: TurnTally | None, + *, + max_segments: int = _MAX_SEGMENTS, +) -> str: + """Render the per-turn accounting line, or ``""`` when there's nothing to say. + + Pure function — no config lookups, no terminal access, no I/O. Gating + (``display.turn_summary``, quiet mode, CLI-only) is the caller's job. + """ + if tally is None: + tally = TurnTally() + + segments: list[str] = [] + for verb in _ordered_verbs(tally): + nouns = tally.verbs[verb] + parts = [_pluralize(count, plural) for plural, count in nouns.items() if count] + if not parts: + continue + segment = f"{verb} {', '.join(parts)}" + if verb == _EDIT_VERB and tally.has_line_deltas: + segment += f" +{tally.lines_added} -{tally.lines_removed}" + segments.append(segment) + + if tally.other_tools: + segments.append(f"called {_pluralize(tally.other_tools, 'tools')}") + + if not segments and tally.total_tools == 0 and elapsed_seconds < _MIN_TOOLLESS_SECONDS: + return "" + + if max_segments > 0 and len(segments) > max_segments: + hidden = len(segments) - max_segments + segments = segments[:max_segments] + [f"+{hidden} more"] + + pieces = [format_elapsed(elapsed_seconds)] + segments + return f"{SUMMARY_PREFIX} " + " · ".join(pieces) + + +def format_token_flow(output_tokens: Any, *, arrow: str = "↓") -> str: + """Render cumulative turn tokens for the live spinner (``↓ 1.2k tok``). + + Returns ``""`` for a non-positive count so the spinner shows nothing + rather than a misleading ``↓ 0 tok`` before the first API response lands. + """ + try: + count = int(output_tokens) + except (TypeError, ValueError): + return "" + if count <= 0: + return "" + if count < 1000: + return f"{arrow} {count} tok" + if count < 1_000_000: + return f"{arrow} {count / 1000:.1f}k tok" + return f"{arrow} {count / 1_000_000:.1f}M tok" diff --git a/agent/usage_pricing.py b/agent/usage_pricing.py index 9825e4a27bd5..04d34d13372c 100644 --- a/agent/usage_pricing.py +++ b/agent/usage_pricing.py @@ -511,15 +511,93 @@ class CostResult: pricing_version="deepseek-pricing-2026-07", ), # Google Gemini + ( + "google", + "gemini-3.6-flash", + ): PricingEntry( + input_cost_per_million=Decimal("1.50"), + output_cost_per_million=Decimal("7.50"), + cache_read_cost_per_million=Decimal("0.15"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/gemini-api/docs/pricing", + pricing_version="google-pricing-2026-07-28", + ), + ( + "google", + "gemini-3.5-flash", + ): PricingEntry( + input_cost_per_million=Decimal("1.50"), + output_cost_per_million=Decimal("9.00"), + cache_read_cost_per_million=Decimal("0.15"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/pricing", + pricing_version="google-pricing-2026-07-07", + ), + ( + "google", + "gemini-3.5-flash-lite", + ): PricingEntry( + input_cost_per_million=Decimal("0.30"), + output_cost_per_million=Decimal("2.50"), + cache_read_cost_per_million=Decimal("0.03"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/gemini-api/docs/pricing", + pricing_version="google-pricing-2026-07-28", + ), + ( + "google", + "gemini-3.1-pro", + ): PricingEntry( + input_cost_per_million=Decimal("2.00"), + output_cost_per_million=Decimal("12.00"), + cache_read_cost_per_million=Decimal("0.20"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/pricing", + pricing_version="google-pricing-2026-07-07", + ), + ( + "google", + "gemini-3.1-flash-lite", + ): PricingEntry( + input_cost_per_million=Decimal("0.25"), + output_cost_per_million=Decimal("1.50"), + cache_read_cost_per_million=Decimal("0.025"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/pricing", + pricing_version="google-pricing-2026-07-07", + ), + ( + "google", + "gemini-3-pro-preview", + ): PricingEntry( + input_cost_per_million=Decimal("2.00"), + output_cost_per_million=Decimal("12.00"), + cache_read_cost_per_million=Decimal("0.20"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/pricing", + pricing_version="google-pricing-2026-07-07", + ), + ( + "google", + "gemini-3-flash-preview", + ): PricingEntry( + input_cost_per_million=Decimal("0.50"), + output_cost_per_million=Decimal("3.00"), + cache_read_cost_per_million=Decimal("0.05"), + source="official_docs_snapshot", + source_url="https://ai.google.dev/pricing", + pricing_version="google-pricing-2026-07-07", + ), ( "google", "gemini-2.5-pro", ): PricingEntry( input_cost_per_million=Decimal("1.25"), output_cost_per_million=Decimal("10.00"), + cache_read_cost_per_million=Decimal("0.125"), source="official_docs_snapshot", source_url="https://ai.google.dev/pricing", - pricing_version="google-pricing-2026-03-16", + pricing_version="google-pricing-2026-07-07", ), ( "google", @@ -527,9 +605,10 @@ class CostResult: ): PricingEntry( input_cost_per_million=Decimal("0.15"), output_cost_per_million=Decimal("0.60"), + cache_read_cost_per_million=Decimal("0.015"), source="official_docs_snapshot", source_url="https://ai.google.dev/pricing", - pricing_version="google-pricing-2026-03-16", + pricing_version="google-pricing-2026-07-07", ), ( "google", @@ -537,9 +616,10 @@ class CostResult: ): PricingEntry( input_cost_per_million=Decimal("0.10"), output_cost_per_million=Decimal("0.40"), + cache_read_cost_per_million=Decimal("0.01"), source="official_docs_snapshot", source_url="https://ai.google.dev/pricing", - pricing_version="google-pricing-2026-03-16", + pricing_version="google-pricing-2026-07-07", ), # AWS Bedrock — pricing per the Bedrock pricing page. # Bedrock charges the same per-token rates as the model provider but @@ -878,6 +958,18 @@ class CostResult: ] del _base_56 +# The direct Gemini provider currently exposes preview IDs for these two +# models. Keep the official snapshot keyed by both their documented stable +# names and the provider's emitted IDs so a catalog selection is billable. +for _alias, _canonical in { + "gemini-3.1-pro-preview": "gemini-3.1-pro", + "gemini-3.1-flash-lite-preview": "gemini-3.1-flash-lite", +}.items(): + _OFFICIAL_DOCS_PRICING[("google", _alias)] = _OFFICIAL_DOCS_PRICING[ + ("google", _canonical) + ] +del _alias, _canonical + def _to_decimal(value: Any) -> Optional[Decimal]: if value is None: @@ -925,11 +1017,17 @@ def resolve_billing_route( return BillingRoute(provider="openai", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") if provider_name in {"minimax", "minimax-cn"}: return BillingRoute(provider=provider_name, model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") - # Vertex AI hosts the same Gemini models as Google AI Studio; price them - # off the gemini official-docs snapshot. Strip the "google/" vendor prefix - # the OpenAI-compat endpoint requires so the pricing key matches. - if provider_name == "vertex" or base_url_host_matches(base_url or "", "aiplatform.googleapis.com"): - return BillingRoute(provider="gemini", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") + # Google AI Studio (Gemini) and Vertex AI host the same Gemini models. + # Price them off the official docs snapshot — the pricing keys are + # keyed on provider='google', so normalize every Google-flavored + # provider name/host onto it. Strip the "google/" vendor prefix the + # Vertex OpenAI-compat endpoint requires so the pricing key matches. + if ( + provider_name in {"google", "gemini", "vertex", "google-gemini", "google-ai-studio", "google-vertex", "vertex-ai"} + or base_url_host_matches(base_url or "", "aiplatform.googleapis.com") + or base_url_host_matches(base_url or "", "generativelanguage.googleapis.com") + ): + return BillingRoute(provider="google", model=model.split("/")[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") if provider_name == "fireworks" or base_url_host_matches(base_url or "", "api.fireworks.ai"): # Fireworks model ids look like accounts/fireworks/models/; # rsplit("/", 1)[-1] yields just which is what the dict keys on. @@ -1146,8 +1244,8 @@ def normalize_usage( output_tokens = _to_int(getattr(response_usage, "completion_tokens", 0)) details = getattr(response_usage, "prompt_tokens_details", None) # Primary: OpenAI-style prompt_tokens_details. Fallback: Anthropic-style - # top-level fields that some OpenAI-compatible proxies (OpenRouter, Cline) - # expose when routing Claude models — without this + # top-level fields that some OpenAI-compatible proxies (OpenRouter, Vercel + # AI Gateway, Cline) expose when routing Claude models — without this # fallback, cache writes are undercounted as 0 and cache reads can be # missed when the proxy only surfaces them at the top level. # Port of cline/cline#10266. diff --git a/agent/verification_evidence.py b/agent/verification_evidence.py index c3154378f5ec..2c8d1f85efa9 100644 --- a/agent/verification_evidence.py +++ b/agent/verification_evidence.py @@ -13,10 +13,11 @@ import sqlite3 import tempfile import threading +from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Optional +from typing import Any, Iterator, Optional from hermes_constants import get_hermes_home @@ -65,13 +66,38 @@ def _connect() -> sqlite3.Connection: path = _db_path() path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path) - apply_wal_with_fallback(conn, db_label="verification_evidence.db") - conn.execute("PRAGMA busy_timeout=5000") conn.row_factory = sqlite3.Row - _ensure_schema(conn) + try: + apply_wal_with_fallback(conn, db_label="verification_evidence.db") + conn.execute("PRAGMA busy_timeout=5000") + _ensure_schema(conn) + except Exception: + # A PRAGMA/DDL failure after a successful connect() must not leak the + # just-opened connection back to the caller. + conn.close() + raise return conn +@contextmanager +def _transaction() -> Iterator[sqlite3.Connection]: + """Open a connection, commit/rollback on exit, and ALWAYS close it. + + ``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the + transaction; they do not close the connection. Using ``with _connect()`` + alone therefore leaks a connection — and its WAL/SHM file descriptors — on + every call, deferring the close to the garbage collector, which over a + long-running process can exhaust ``RLIMIT_NOFILE`` (the cron-ledger sibling + of this bug was #69567 / PR #69594). + """ + conn = _connect() + try: + with conn: + yield conn + finally: + conn.close() + + def _ensure_schema(conn: sqlite3.Connection) -> None: conn.execute( """ @@ -454,7 +480,7 @@ def record_terminal_result( created_at = _utc_now() with _DB_LOCK: - with _connect() as conn: + with _transaction() as conn: cur = conn.execute( """ INSERT INTO verification_events( @@ -520,7 +546,7 @@ def mark_workspace_edited( edited_at = _utc_now() with _DB_LOCK: - with _connect() as conn: + with _transaction() as conn: row = conn.execute( """ SELECT changed_paths_json FROM verification_state @@ -570,7 +596,7 @@ def verification_status( sid = str(session_id or "default") root = str(facts.get("root") or Path(cwd or ".").resolve()) with _DB_LOCK: - with _connect() as conn: + with _transaction() as conn: state = conn.execute( """ SELECT last_event_id, last_edit_at, changed_paths_json diff --git a/agent/verification_stop.py b/agent/verification_stop.py index 1f68b1aace55..dc66944883c9 100644 --- a/agent/verification_stop.py +++ b/agent/verification_stop.py @@ -72,64 +72,24 @@ def _filter_verifiable_paths(paths: Iterable[str]) -> list[str]: return [p for p in paths if p and not _is_non_code_path(p)] -# Session identities (platform or source) that are NOT human conversational -# messaging surfaces: interactive coding surfaces (CLI, TUI, desktop, codex, -# local, gateway) and programmatic callers (API server, webhooks, tools). -# Verify-on-stop stays ON by default for these. Any other resolved gateway -# platform is a conversational messaging surface (Telegram, Discord, WhatsApp, -# Signal, Slack, etc.) where the verification narrative would reach a human as -# chat noise, so it defaults OFF. Mirrors LOCAL_SESSION_SOURCE_IDS in -# apps/desktop/src/lib/session-source.ts; keep roughly in sync when adding a -# local or programmatic surface. Default-deny by design: an unrecognized -# identity is treated as messaging (OFF) so a new chat platform never leaks the -# verification receipt before this set is updated. -_NON_MESSAGING_SESSION_SURFACES = frozenset( - { - "", - "cli", - "codex", - "desktop", - "gateway", - "local", - "tui", - "tool", - "api_server", - "webhook", - "msgraph_webhook", - } -) - - def _session_is_messaging_surface() -> bool: - """Return whether this turn is delivered over a human messaging channel. - - The gateway binds the platform value (e.g. ``telegram``) to - ``HERMES_SESSION_PLATFORM``; the CLI and TUI set ``HERMES_SESSION_SOURCE`` - (e.g. ``cli``, ``tui``) instead. Both are consulted via the session-context - helper (with an ``os.environ`` fallback), alongside the ``HERMES_PLATFORM`` - override, matching the sibling platform resolution in - ``agent/skill_commands.py`` and ``agent/prompt_builder.py``. A turn is a - messaging surface when a resolved identity is present and is not a known - non-messaging surface. + """Whether this turn is delivered over a human messaging channel. + + Verify-on-stop defaults ON for the interactive coding surfaces and + programmatic callers, and OFF on a conversational platform (Telegram, + Discord, Slack, ...) where the verification narrative reaches a human as + chat noise. The surface classification itself is shared with the other + consumers of this distinction — see + ``gateway.session_context.session_is_messaging_surface``. """ try: - from gateway.session_context import get_session_env + from gateway.session_context import session_is_messaging_surface - platform = ( - os.getenv("HERMES_PLATFORM") - or get_session_env("HERMES_SESSION_PLATFORM", "") - ) - source = get_session_env("HERMES_SESSION_SOURCE", "") + return session_is_messaging_surface() except Exception: - platform = os.getenv("HERMES_PLATFORM", "") or os.environ.get( - "HERMES_SESSION_PLATFORM", "" - ) - source = os.environ.get("HERMES_SESSION_SOURCE", "") - for identity in (platform, source): - identity = str(identity or "").strip().lower() - if identity and identity not in _NON_MESSAGING_SESSION_SURFACES: - return True - return False + # The gateway package is unreachable, so there is no messaging channel + # to be on. Reporting a local surface keeps verify-on-stop enabled. + return False def verify_on_stop_enabled(config: dict[str, Any] | None = None) -> bool: @@ -149,9 +109,9 @@ def verify_on_stop_enabled(config: dict[str, Any] | None = None) -> bool: return env.strip().lower() not in {"0", "false", "no", "off"} if config is None: try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - config = load_config() + config = load_config_readonly() except Exception: config = {} agent_cfg = (config or {}).get("agent") if isinstance(config, dict) else None diff --git a/agent/video_gen_registry.py b/agent/video_gen_registry.py index c4d28e39ed4b..d78babfc9bb8 100644 --- a/agent/video_gen_registry.py +++ b/agent/video_gen_registry.py @@ -84,9 +84,9 @@ def get_active_provider() -> Optional[VideoGenProvider]: """ configured: Optional[str] = None try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - cfg = load_config() + cfg = load_config_readonly() section = cfg.get("video_gen") if isinstance(cfg, dict) else None if isinstance(section, dict): raw = section.get("provider") diff --git a/agent/web_search_registry.py b/agent/web_search_registry.py index 45832cb5488d..dd46eb681186 100644 --- a/agent/web_search_registry.py +++ b/agent/web_search_registry.py @@ -98,9 +98,9 @@ def get_provider(name: str) -> Optional[WebSearchProvider]: def _read_config_key(*path: str) -> Optional[str]: """Resolve a dotted config key from ``config.yaml``. Returns None on miss.""" try: - from hermes_cli.config import load_config + from hermes_cli.config import load_config_readonly - cfg = load_config() + cfg = load_config_readonly() cur = cfg for segment in path: if not isinstance(cur, dict): diff --git a/apps/bootstrap-installer/package.json b/apps/bootstrap-installer/package.json index 32c3e7f8c3e7..5e72fb283f1a 100644 --- a/apps/bootstrap-installer/package.json +++ b/apps/bootstrap-installer/package.json @@ -19,41 +19,33 @@ "fix": "npm run lint:fix" }, "dependencies": { - "@nous-research/ui": "0.16.0", - "@tailwindcss/vite": "^4.2.4", - "@tailwindcss/typography": "^0.5.19", - "@tauri-apps/api": "^2.0.0", - "@tauri-apps/plugin-dialog": "^2.0.0", - "@tauri-apps/plugin-opener": "^2.0.0", - "@tauri-apps/plugin-process": "^2.0.0", - "@tauri-apps/plugin-shell": "^2.0.0", - "@vscode/codicons": "^0.0.45", - "class-variance-authority": "^0.7.1", - "clsx": "^2.1.1", - "katex": "^0.16.45", - "lucide-react": "^0.577.0", - "nanostores": "^1.3.0", - "radix-ui": "^1.4.3", - "react": "^19.2.4", - "react-dom": "^19.2.4", - "tailwind-merge": "^3.5.0", - "tailwindcss": "^4.2.1", - "tw-shimmer": "^0.4.11" + "@nous-research/ui": "0.18.2", + "@tailwindcss/typography": "0.5.20", + "@tailwindcss/vite": "4.3.3", + "@tauri-apps/api": "2.11.1", + "@tauri-apps/plugin-dialog": "2.7.1", + "@tauri-apps/plugin-opener": "2.5.4", + "@tauri-apps/plugin-process": "2.3.1", + "@tauri-apps/plugin-shell": "2.3.5", + "@vscode/codicons": "0.0.45", + "class-variance-authority": "0.7.1", + "clsx": "2.1.1", + "katex": "0.16.47", + "lucide-react": "0.577.0", + "nanostores": "1.4.0", + "radix-ui": "1.6.7", + "react": "19.2.7", + "react-dom": "19.2.7", + "tailwind-merge": "3.6.0", + "tailwindcss": "4.3.3", + "tw-shimmer": "0.4.12" }, "devDependencies": { - "@eslint/js": "^9.39.4", - "@tauri-apps/cli": "^2.0.0", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.2", - "eslint": "^9.39.4", - "eslint-plugin-perfectionist": "^5.9.0", - "eslint-plugin-react": "^7.37.5", - "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-unused-imports": "^4.4.1", - "globals": "^17.4.0", - "typescript": "^6.0.3", - "typescript-eslint": "^8.56.1", - "vite": "^8.0.16" + "@tauri-apps/cli": "2.11.4", + "@types/react": "19.2.17", + "@types/react-dom": "19.2.3", + "@vitejs/plugin-react": "6.0.3", + "typescript": "6.0.3", + "vite": "8.2.0" } } diff --git a/apps/bootstrap-installer/src-tauri/Cargo.toml b/apps/bootstrap-installer/src-tauri/Cargo.toml index fe65ff9aa7be..d6b012aed674 100644 --- a/apps/bootstrap-installer/src-tauri/Cargo.toml +++ b/apps/bootstrap-installer/src-tauri/Cargo.toml @@ -66,6 +66,10 @@ windows-sys = { version = "0.59", features = [ "Win32_UI_WindowsAndMessaging", ] } +# Signal-0 liveness probe for the update-lock marker owner (update.rs). +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [profile.release] # A 5-10MB signed installer is the goal. LTO + size-opt + single codegen unit. panic = "abort" diff --git a/apps/bootstrap-installer/src-tauri/src/bootstrap.rs b/apps/bootstrap-installer/src-tauri/src/bootstrap.rs index 1d70ec59a611..f78b26134e41 100644 --- a/apps/bootstrap-installer/src-tauri/src/bootstrap.rs +++ b/apps/bootstrap-installer/src-tauri/src/bootstrap.rs @@ -12,11 +12,11 @@ //! 4. Worker iterates stages, calling `install.ps1 -Stage NAME -NonInteractive -Json`. //! 5. On success → `complete`. On any stage failure → `failed`. On cancel → `failed`. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::Instant; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Emitter, State}; use tokio::sync::{mpsc, Mutex}; @@ -260,6 +260,107 @@ pub(crate) fn hermes_is_installed(install_root: &std::path::Path) -> bool { && resolve_hermes_desktop_exe(install_root).is_some() } +fn resolve_marker_commit(install_root: &Path, pin: &Pin) -> Option { + if let Some(commit) = pin + .commit + .as_ref() + .filter(|commit| !commit.trim().is_empty()) + { + return Some(commit.clone()); + } + + let output = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(install_root) + .output() + .ok()?; + if !output.status.success() { + return None; + } + + let commit = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if commit.is_empty() { + None + } else { + Some(commit) + } +} + +fn write_bootstrap_complete_marker(install_root: &Path, pin: &Pin) -> Result { + use std::io::Write; + + let marker_path = crate::paths::likely_bootstrap_marker(install_root); + if let Some(parent) = marker_path.parent() { + std::fs::create_dir_all(parent).with_context(|| { + format!( + "could not create bootstrap marker directory {}", + parent.display() + ) + })?; + } + + let completed_at_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default(); + let marker = serde_json::json!({ + "schemaVersion": 1, + "pinnedCommit": resolve_marker_commit(install_root, pin), + "pinnedBranch": pin.branch.clone(), + "completedAtUnix": completed_at_unix, + }); + let mut body = serde_json::to_vec_pretty(&marker)?; + body.push(b'\n'); + + // Atomic publish (temp sibling + flush + rename), matching Electron's + // writeFileAtomic(). hermes_is_installed() only checks existence, so a + // partial direct write would incorrectly enable the launcher fast path. + let tmp_path = install_root.join(".hermes-bootstrap-complete.tmp"); + { + let mut file = std::fs::File::create(&tmp_path).with_context(|| { + format!( + "could not create temp bootstrap marker {}", + tmp_path.display() + ) + })?; + file.write_all(&body).with_context(|| { + format!( + "could not write temp bootstrap marker {}", + tmp_path.display() + ) + })?; + file.sync_all().with_context(|| { + format!( + "could not flush temp bootstrap marker {}", + tmp_path.display() + ) + })?; + } + // Windows rename fails if the destination already exists; drop any prior + // marker first so a re-run can still publish a fresh payload. + if marker_path.exists() { + std::fs::remove_file(&marker_path).with_context(|| { + format!( + "could not replace existing bootstrap marker {}", + marker_path.display() + ) + })?; + } + if let Err(err) = std::fs::rename(&tmp_path, &marker_path) { + let _ = std::fs::remove_file(&tmp_path); + return Err(err).with_context(|| { + format!( + "could not publish bootstrap marker {} → {}", + tmp_path.display(), + marker_path.display() + ) + }); + } + + tracing::info!(path = %marker_path.display(), "bootstrap marker written"); + Ok(marker) +} + /// Spawn the already-built desktop app, detached. Returns Err if no built app /// exists or the spawn fails, so the caller can fall back to showing the /// installer UI. @@ -644,6 +745,23 @@ async fn run_bootstrap( .unwrap_or_else(|| crate::paths::hermes_home().to_string_lossy().into_owned()); let install_root = PathBuf::from(&hermes_home).join("hermes-agent"); + // Marker publish is terminal for this run: a write failure must emit Failed + // so the UI leaves the progress state (it does not poll get_bootstrap_status). + let marker = match write_bootstrap_complete_marker(&install_root, &pin) { + Ok(marker) => marker, + Err(err) => { + let msg = format!("write bootstrap marker failed: {err:#}"); + emit_event( + &app, + BootstrapEvent::Failed { + stage: None, + error: msg.clone(), + }, + ); + return Err(anyhow!(msg)); + } + }; + // Copy ourselves to HERMES_HOME/hermes-setup.exe so the desktop app can // re-invoke us with `--update` and shortcuts have a stable target. This is // a one-shot install concern; an `--update` re-invocation no-ops because @@ -660,10 +778,7 @@ async fn run_bootstrap( &app, BootstrapEvent::Complete { install_root: install_root.to_string_lossy().into_owned(), - marker: Some(serde_json::json!({ - "pinnedCommit": pin.commit, - "pinnedBranch": pin.branch, - })), + marker: Some(marker), }, ); @@ -903,4 +1018,103 @@ mod tests { ); let _ = std::fs::remove_dir_all(&root); } + + #[test] + fn bootstrap_complete_marker_uses_desktop_compatible_schema() { + let root = unique_tmp_dir("marker-schema"); + let pin = Pin { + commit: Some("abcdef1234567890".to_string()), + branch: Some("main".to_string()), + }; + + let marker = + write_bootstrap_complete_marker(&root, &pin).expect("marker write should succeed"); + let marker_path = root.join(".hermes-bootstrap-complete"); + let from_disk: serde_json::Value = + serde_json::from_slice(&std::fs::read(&marker_path).unwrap()).unwrap(); + + assert_eq!(marker, from_disk); + assert_eq!(from_disk["schemaVersion"], 1); + assert_eq!(from_disk["pinnedCommit"], "abcdef1234567890"); + assert_eq!(from_disk["pinnedBranch"], "main"); + assert!( + from_disk["completedAtUnix"].as_u64().is_some(), + "marker must carry a completion timestamp" + ); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn bootstrap_complete_marker_is_published_atomically() { + let root = unique_tmp_dir("marker-atomic"); + make_release_tree(&root); + let pin = Pin { + commit: Some("abcdef1234567890".to_string()), + branch: Some("main".to_string()), + }; + + write_bootstrap_complete_marker(&root, &pin).expect("marker write should succeed"); + + let marker_path = root.join(".hermes-bootstrap-complete"); + let tmp_path = root.join(".hermes-bootstrap-complete.tmp"); + assert!( + marker_path.is_file(), + "final marker must exist after atomic publish" + ); + assert!( + !tmp_path.exists(), + "temp sibling must not remain after atomic publish" + ); + assert!( + hermes_is_installed(&root), + "atomically published marker must enable the installer fast path" + ); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn hermes_is_installed_treats_marker_existence_as_sufficient() { + // Documents why write_bootstrap_complete_marker must publish atomically: + // the launcher predicate only checks existence, so a partial/corrupt + // final marker would still enable the fast path. + let root = unique_tmp_dir("marker-existence-only"); + make_release_tree(&root); + std::fs::write(root.join(".hermes-bootstrap-complete"), b"").unwrap(); + + assert!( + hermes_is_installed(&root), + "empty/partial marker content still counts as installed" + ); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn marker_write_failure_leaves_no_final_marker() { + // install_root is a regular file → create_dir_all on its path fails + // before any marker bytes are published under the final name. + let base = unique_tmp_dir("marker-fail"); + let not_a_dir = base.join("not-a-dir"); + std::fs::write(¬_a_dir, b"not a directory").unwrap(); + let pin = Pin { + commit: Some("abcdef1234567890".to_string()), + branch: Some("main".to_string()), + }; + + let err = write_bootstrap_complete_marker(¬_a_dir, &pin) + .expect_err("marker write against a non-directory root must fail"); + let msg = format!("{err:#}"); + assert!( + msg.contains("bootstrap marker"), + "error should mention the marker path: {msg}" + ); + assert!( + !not_a_dir.join(".hermes-bootstrap-complete").exists(), + "failed write must not leave a final marker that enables the fast path" + ); + assert!( + !not_a_dir.join(".hermes-bootstrap-complete.tmp").exists(), + "failed write must not leave a temp marker sibling either" + ); + let _ = std::fs::remove_dir_all(&base); + } } diff --git a/apps/bootstrap-installer/src-tauri/src/paths.rs b/apps/bootstrap-installer/src-tauri/src/paths.rs index 7c64c91cf6ef..3a7b1b0dbf5f 100644 --- a/apps/bootstrap-installer/src-tauri/src/paths.rs +++ b/apps/bootstrap-installer/src-tauri/src/paths.rs @@ -98,6 +98,12 @@ pub fn update_in_progress_marker() -> PathBuf { /// that path), where copying onto ourselves would be a Windows sharing /// violation. Best-effort: a failure here must not fail the install, so the /// caller logs and continues. +/// +/// NOTE: because of that no-op, a user's staged installer is only ever written +/// by a full install/repair. Every later `--update` runs the ORIGINAL binary, +/// so an installer-protocol change can strand the whole installed base on a +/// binary that predates it (see `restage_from_checkout`, which repairs this +/// from the freshly-updated checkout). pub fn copy_self_to_hermes_home() -> std::io::Result<()> { let src = std::env::current_exe()?; let dest = installer_dest(); @@ -149,8 +155,8 @@ fn repair_macos_installer_helper(path: &Path) { #[cfg(not(target_os = "macos"))] fn repair_macos_installer_helper(_path: &Path) {} -/// Where install.ps1 writes the bootstrap-complete marker (existence-only file -/// the Electron app also checks). Per main.ts: +/// Where the bootstrap-complete marker lives (existence-only for the Rust +/// installer fast path; JSON schema-checked by the Electron app). Per main.ts: /// const BOOTSTRAP_COMPLETE_MARKER = path.join(ACTIVE_HERMES_ROOT, '.hermes-bootstrap-complete') /// We don't always know ACTIVE_HERMES_ROOT until install.ps1 reports it, so /// this is a probe helper, not a definitive path. diff --git a/apps/bootstrap-installer/src-tauri/src/update.rs b/apps/bootstrap-installer/src-tauri/src/update.rs index be4884ab2634..4614bacf1cb2 100644 --- a/apps/bootstrap-installer/src-tauri/src/update.rs +++ b/apps/bootstrap-installer/src-tauri/src/update.rs @@ -107,16 +107,110 @@ pub async fn start_update(app: AppHandle) -> Result<(), String> { /// future desktop launches. The marker payload is `{pid}\n{started_at_unix}` /// so the desktop's launch gate can detect a stale marker (dead PID / past a /// hard ceiling) and self-heal rather than wait forever. +/// +/// The marker is also the cross-process update lock: `hermes update` claims +/// the same file (see `hermes_cli/update_lock.py`) so a dashboard-spawned +/// update and this updater can't mutate one checkout at the same time. +/// `acquire` therefore REFUSES when a live foreign owner holds it rather than +/// overwriting — the pre-fix clobber is what let a dashboard `hermes update` +/// keep running while install-mode bootstrap rewrote the tree underneath it. struct UpdateMarkerGuard { path: PathBuf, + /// False when a live foreign updater already owns the marker: we hold no + /// claim, so `Drop` must not delete their marker. + owned: bool, +} + +/// Never treat a marker older than this as a live update. Mirrors +/// UPDATE_MARKER_MAX_AGE_MS in apps/desktop/electron/update-marker.ts and +/// UPDATE_MARKER_MAX_AGE_SECONDS in hermes_cli/update_lock.py — all three read +/// this one file, so a shorter ceiling in any of them would steal a lock the +/// others still consider live. +const UPDATE_MARKER_MAX_AGE_SECS: u64 = 20 * 60; + +/// The pid + age of a confirmed-live update holding the marker. +struct MarkerOwner { + pid: u32, + age_secs: u64, +} + +/// Read the marker and report a live *foreign* owner, if any. `None` for every +/// "no live update" case — absent, unreadable, malformed, dead pid, past the +/// ceiling, or a marker whose pid is **this** process — matching +/// `readLiveUpdateMarker` in the Electron gate. Never panics. +/// +/// Self-PID is treated as non-ownership on purpose (#74761): since #50238 the +/// desktop pre-writes this marker with the spawned updater's pid before the +/// updater reaches `acquire`. Without the exclusion, `acquire` sees a live +/// owner that is itself and aborts ("Another Hermes update is already +/// running"), then the desktop relaunches and retries forever. A foreign live +/// pid (e.g. a dashboard-spawned `hermes update`) still blocks. +fn live_marker_owner(path: &Path) -> Option { + let raw = std::fs::read_to_string(path).ok()?; + let mut lines = raw.lines(); + let pid: u32 = lines.next()?.trim().parse().ok()?; + let started_at: u64 = lines.next().unwrap_or("").trim().parse().unwrap_or(0); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let age_secs = now.saturating_sub(started_at); + if age_secs > UPDATE_MARKER_MAX_AGE_SECS || !pid_is_alive(pid) { + return None; + } + // Desktop `writeUpdateMarker(hermesHome, child.pid)` races ahead of us; + // adopt that pre-claim rather than refusing our own marker. + if pid == std::process::id() { + return None; + } + Some(MarkerOwner { pid, age_secs }) +} + +/// True when a process with `pid` currently exists. +#[cfg(windows)] +fn pid_is_alive(pid: u32) -> bool { + use windows_sys::Win32::Foundation::{CloseHandle, STILL_ACTIVE}; + use windows_sys::Win32::System::Threading::{ + GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + // Either the pid is gone or we lack rights to open it. A pid we + // can't inspect is treated as dead so an unopenable straggler + // can't wedge every future update. + return false; + } + let mut code: u32 = 0; + let ok = GetExitCodeProcess(handle, &mut code); + CloseHandle(handle); + ok != 0 && code == STILL_ACTIVE as u32 + } +} + +#[cfg(not(windows))] +fn pid_is_alive(pid: u32) -> bool { + // signal 0 delivers nothing; it only probes existence/permission. + // ESRCH => dead. EPERM => alive but owned by another user. + let rc = unsafe { libc::kill(pid as libc::pid_t, 0) }; + if rc == 0 { + return true; + } + std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) } impl UpdateMarkerGuard { - /// Write the marker. Best-effort: a write failure must NOT abort the - /// update (the gate degrades to "no marker => proceed", i.e. exactly the - /// pre-fix behavior), so we log and carry on with a guard that still - /// attempts cleanup of whatever may exist at the path. - fn acquire(path: PathBuf) -> Self { + /// Claim the marker, or report the live updater that already owns it. + /// + /// Writing is best-effort: a write failure must NOT abort the update (the + /// gate degrades to "no marker => proceed", i.e. exactly the pre-marker + /// behavior), so we log and carry on with a guard that still attempts + /// cleanup of whatever may exist at the path. + fn acquire(path: PathBuf) -> Result { + if let Some(owner) = live_marker_owner(&path) { + return Err(owner); + } let pid = std::process::id(); let started_at = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -128,20 +222,35 @@ impl UpdateMarkerGuard { if let Err(err) = std::fs::write(&path, format!("{pid}\n{started_at}")) { tracing::warn!(?path, %err, "could not write update-in-progress marker"); } - Self { path } + Ok(Self { path, owned: true }) } -} -impl Drop for UpdateMarkerGuard { - fn drop(&mut self) { + /// Release the marker as soon as every mutating stage has completed. + /// + /// The updater still owns a Tauri/Cocoa event loop while it relaunches the + /// desktop, and that loop can outlive `app.exit(0)`. Relying on `Drop` + /// alone therefore leaves a *successful* update looking active — a live + /// pid holding a fresh marker — which blocks desktop startup and every + /// other updater for the full age ceiling. Idempotent: `Drop` still runs + /// and tolerates an already-removed marker. + fn complete(&self) { + if !self.owned { + return; + } if let Err(err) = std::fs::remove_file(&self.path) { if err.kind() != std::io::ErrorKind::NotFound { - tracing::warn!(path = ?self.path, %err, "could not remove update-in-progress marker"); + tracing::warn!(path = ?self.path, %err, "could not remove completed update marker"); } } } } +impl Drop for UpdateMarkerGuard { + fn drop(&mut self) { + self.complete(); + } +} + async fn run_update(app: AppHandle) -> Result<()> { let hermes_home = crate::paths::hermes_home(); let install_root = hermes_home.join("hermes-agent"); @@ -152,7 +261,39 @@ async fn run_update(app: AppHandle) -> Result<()> { // it, that backend re-locks the venv shim, our `force_kill_other_hermes` // straggler-cleanup kills it, and the relaunch/kill cycle loops. The guard // removes the marker on every exit path (incl. early returns / panics). - let _update_marker = UpdateMarkerGuard::acquire(crate::paths::update_in_progress_marker()); + // + // The same marker is the cross-process update lock (hermes_cli/ + // update_lock.py claims it too), so a live foreign owner means another + // updater — most often a dashboard-spawned `hermes update` — is already + // mutating this checkout. Refuse instead of running a second one over it. + let _update_marker = match UpdateMarkerGuard::acquire( + crate::paths::update_in_progress_marker(), + ) { + Ok(guard) => guard, + Err(owner) => { + let mins = owner.age_secs / 60; + let secs = owner.age_secs % 60; + let elapsed = if mins > 0 { + format!("{mins}m {secs}s") + } else { + format!("{secs}s") + }; + let msg = format!( + "Another Hermes update is already running (PID {}, started {} ago). \ + Wait for it to finish, or close the window or dashboard tab that \ + started it, then try again.", + owner.pid, elapsed + ); + emit( + &app, + BootstrapEvent::Failed { + stage: None, + error: msg.clone(), + }, + ); + return Err(anyhow!(msg)); + } + }; let update_branch = update_branch_from_args(std::env::args().skip(1)) .or_else(|| option_env_string("BUILD_PIN_BRANCH")) @@ -453,6 +594,12 @@ async fn run_update(app: AppHandle) -> Result<()> { marker: None, }, ); + // Every install-tree mutation is finished. Release the lock BEFORE the + // relaunch: this process can stay wedged in its native event loop even + // after a successful app.exit(), and a live pid on a fresh marker would + // make a completed update look active — blocking desktop startup and + // every other updater until the age ceiling expires. + _update_marker.complete(); if let Some(target_app) = launch_target { if let Err(err) = launch_macos_app_and_exit(&app, &target_app).await { @@ -477,9 +624,26 @@ async fn run_update(app: AppHandle) -> Result<()> { ); } + // The launch helpers normally request exit themselves, but their failure + // paths must still close a successful updater. A native event loop can + // ignore that graceful request, so arm a process-exit fallback now that + // all update state and the marker have been settled. + exit_after_success(&app); Ok(()) } +/// Ask the app to exit, with a hard `process::exit` fallback for a native +/// event loop that ignores the graceful request. Without it a finished updater +/// can linger as a live pid forever. +fn exit_after_success(app: &AppHandle) { + std::thread::spawn(|| { + std::thread::sleep(std::time::Duration::from_secs(3)); + tracing::warn!("graceful updater exit timed out; forcing process exit"); + std::process::exit(0); + }); + app.exit(0); +} + /// Poll until the venv shim AND packaged desktop app bundle are no longer locked /// (Windows) or a bounded timeout elapses. On non-Windows this is a short fixed /// grace since file locking isn't the failure mode there. @@ -744,6 +908,17 @@ fn update_child_env(install_root: &Path) -> Vec<(String, OsString)> { // a frozen stage, and users cancel a healthy update. Force line-by-line // output instead. envs.push(("PYTHONUNBUFFERED".to_string(), OsString::from("1"))); + // We hold the update-in-progress marker for this whole run, and the + // `hermes update` child claims that SAME lock (hermes_cli/update_lock.py). + // Name our pid so the child recognizes the live holder as its own + // orchestrator and runs under our claim — without this every GUI update + // refuses its parent's marker with exit 2 ("Hermes is still running") + // and no number of retries can ever succeed. Keep the variable name in + // sync with HANDOFF_PID_ENV in hermes_cli/update_lock.py. + envs.push(( + "HERMES_UPDATE_HANDOFF_PID".to_string(), + OsString::from(std::process::id().to_string()), + )); if let Some(path) = path_with_prepended_entries(&[ hermes_home.join("node").join("bin"), venv_bin_dir(install_root), @@ -1067,6 +1242,17 @@ mod tests { ); } + #[test] + fn update_child_env_names_our_pid_for_the_lock_handoff() { + let envs = update_child_env(Path::new("/x/hermes-agent")); + assert!( + envs.iter().any(|(k, v)| k == "HERMES_UPDATE_HANDOFF_PID" + && v.to_str() == Some(std::process::id().to_string().as_str())), + "the hermes update child claims the same marker we hold; without our pid \ + it refuses its own parent's lock and every GUI update dead-ends on exit 2" + ); + } + #[test] fn lock_probe_paths_include_desktop_app_payload() { let root = Path::new("/x/hermes-agent"); @@ -1102,7 +1288,8 @@ mod tests { let marker = dir.join(".hermes-update-in-progress"); { - let _g = UpdateMarkerGuard::acquire(marker.clone()); + let _g = UpdateMarkerGuard::acquire(marker.clone()) + .unwrap_or_else(|_| panic!("no live owner => acquire must succeed")); assert!(marker.exists(), "marker must exist while the guard is held"); let body = std::fs::read_to_string(&marker).unwrap(); let pid_line = body.lines().next().unwrap(); @@ -1127,7 +1314,8 @@ mod tests { std::fs::create_dir_all(&dir).unwrap(); let marker = dir.join(".hermes-update-in-progress"); - let guard = UpdateMarkerGuard::acquire(marker.clone()); + let guard = UpdateMarkerGuard::acquire(marker.clone()) + .unwrap_or_else(|_| panic!("no live owner => acquire must succeed")); // Simulate an external cleanup (e.g. the desktop pruned a marker it // judged stale) before our guard drops — Drop must not panic. std::fs::remove_file(&marker).unwrap(); @@ -1137,6 +1325,166 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// Spawn a short-lived sibling process whose pid stands in for a foreign + /// updater. Same-process double-acquire no longer models contention: since + /// #74761 `live_marker_owner` treats our own pid as adoptable (desktop + /// pre-writes it), so a second acquire in *this* process would succeed. + fn spawn_foreign_holder() -> std::process::Child { + #[cfg(windows)] + { + std::process::Command::new("timeout") + .args(["/t", "30", "/nobreak"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn foreign marker holder") + } + #[cfg(not(windows))] + { + std::process::Command::new("sleep") + .arg("30") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn foreign marker holder") + } + } + + #[test] + fn acquire_refuses_while_a_live_updater_owns_the_marker() { + let dir = unique_tmp_dir("marker-contended"); + std::fs::create_dir_all(&dir).unwrap(); + let marker = dir.join(".hermes-update-in-progress"); + + // A live *foreign* updater holds it. We must NOT clobber the marker and + // run concurrently over the same checkout — that race is what let a + // dashboard `hermes update` and install-mode bootstrap mutate one tree + // at once. Own-pid markers are adoptable (#74761), so the foreign pid + // must be a real sibling process. + let mut foreign = spawn_foreign_holder(); + let foreign_pid = foreign.id(); + let started_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + std::fs::write(&marker, format!("{foreign_pid}\n{started_at}")).unwrap(); + + let owner = UpdateMarkerGuard::acquire(marker.clone()) + .err() + .expect("acquire must be refused while a foreign updater is live"); + assert_eq!(owner.pid, foreign_pid); + + // The refused guard must not delete the live owner's marker. + assert!(marker.exists(), "refused acquire must leave the marker intact"); + let _ = foreign.kill(); + let _ = foreign.wait(); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn acquire_adopts_a_marker_prewritten_with_our_own_pid() { + // #74761: desktop writeUpdateMarker(hermesHome, child.pid) races ahead + // of UpdateMarkerGuard::acquire. The marker names US; refusing it made + // every in-app desktop update loop forever. Adopt and rewrite. + let dir = unique_tmp_dir("marker-own-pid"); + std::fs::create_dir_all(&dir).unwrap(); + let marker = dir.join(".hermes-update-in-progress"); + + let started_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + .saturating_sub(2); + std::fs::write(&marker, format!("{}\n{started_at}", std::process::id())).unwrap(); + + let guard = UpdateMarkerGuard::acquire(marker.clone()).unwrap_or_else(|owner| { + panic!( + "own-pid pre-write must be adoptable, got foreign owner pid={}", + owner.pid + ) + }); + assert!(marker.exists(), "adopted guard must own the marker"); + let body = std::fs::read_to_string(&marker).unwrap(); + assert_eq!( + body.lines().next().unwrap().trim().parse::().unwrap(), + std::process::id(), + "acquire rewrites the marker with our pid + fresh started_at" + ); + drop(guard); + assert!( + !marker.exists(), + "Drop must still clear the marker we adopted" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn acquire_reclaims_a_marker_owned_by_a_dead_pid() { + let dir = unique_tmp_dir("marker-dead-pid"); + std::fs::create_dir_all(&dir).unwrap(); + let marker = dir.join(".hermes-update-in-progress"); + + // pid 1 exists everywhere, so fabricate a dead one: a very large pid + // that no live process owns. A crashed updater must never wedge every + // future update. + let started_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + std::fs::write(&marker, format!("4294967294\n{started_at}")).unwrap(); + + let guard = UpdateMarkerGuard::acquire(marker.clone()) + .unwrap_or_else(|_| panic!("a dead owner must not block acquisition")); + let body = std::fs::read_to_string(&marker).unwrap(); + assert_eq!( + body.lines().next().unwrap().trim().parse::().unwrap(), + std::process::id(), + "reclaiming rewrites the marker with our pid" + ); + drop(guard); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn acquire_reclaims_a_marker_past_the_age_ceiling() { + let dir = unique_tmp_dir("marker-stale-age"); + std::fs::create_dir_all(&dir).unwrap(); + let marker = dir.join(".hermes-update-in-progress"); + + // Our own (live) pid, but started well past the ceiling: a wedged + // updater must not hold the lock forever. + let long_ago = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + .saturating_sub(UPDATE_MARKER_MAX_AGE_SECS + 60); + std::fs::write(&marker, format!("{}\n{long_ago}", std::process::id())).unwrap(); + + let guard = UpdateMarkerGuard::acquire(marker.clone()) + .unwrap_or_else(|_| panic!("a marker past the ceiling must be reclaimable")); + drop(guard); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn completed_update_releases_marker_before_guard_drop() { + let dir = unique_tmp_dir("marker-complete"); + std::fs::create_dir_all(&dir).unwrap(); + let marker = dir.join(".hermes-update-in-progress"); + + let guard = UpdateMarkerGuard::acquire(marker.clone()) + .unwrap_or_else(|_| panic!("no live owner => acquire must succeed")); + guard.complete(); + + assert!( + !marker.exists(), + "a successful update must unblock desktop startup before relaunch/exit" + ); + drop(guard); + assert!(!marker.exists(), "Drop stays idempotent after completion"); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn parses_update_branch_from_space_or_equals_args() { assert_eq!( diff --git a/apps/desktop/DESIGN.md b/apps/desktop/DESIGN.md index 53075c5d49bb..e2f3614022a3 100644 --- a/apps/desktop/DESIGN.md +++ b/apps/desktop/DESIGN.md @@ -91,10 +91,11 @@ for call-site shadow or border inventions. | Token | Use | | --- | --- | | `--ui-stroke-primary…quaternary` | hairlines, in descending strength | -| `--ui-stroke-tertiary` | the default in-panel divider / list hairline | +| `--ui-stroke-tertiary` | the default in-panel divider / list hairline — and every bordered surface in the transcript | | `--stroke-nous` | the overlay hairline (pairs with `shadow-nous`) | | `--ui-text-primary / -secondary / -tertiary` | text hierarchy | | `--ui-bg-quaternary` | soft control fill (secondary button) | +| `--ui-widget-surface-background` | fill for inline chat widgets (`WIDGET_SHELL_CLASS`) | | `--chrome-action-hover` | hover fill for quiet controls | | `--theme-primary`, `--ui-accent` | brand/accent | @@ -117,20 +118,31 @@ that sit inside a heading/sentence; replaces `h-auto px-0 py-0`), `micro` (status-stack/table-footers), and the icon family `icon` / `icon-xs` / `icon-sm` / `icon-lg` / `icon-titlebar`. -**Icon-only buttons must have a tooltip.** Every button with an `icon*` size -carries no visible text label, so it must be wrapped in `` -with a descriptive label (matching the button's `aria-label`). Never use the -native HTML `title=` attribute — it's unstyled, delayed (~500ms OS default), -and visually inconsistent with the instant themed `Tip`. An enforcement test -(`src/components/ui/__tests__/no-native-title.test.ts`) fails on any ` - {onRemove && ( + <> + +
- )} -
- + {onRemove && ( + + )} + + + {lightboxSrc && ( + + )} + ) } diff --git a/apps/desktop/src/app/chat/composer/composer-utils.test.ts b/apps/desktop/src/app/chat/composer/composer-utils.test.ts index 4df8463ba2de..062b25677385 100644 --- a/apps/desktop/src/app/chat/composer/composer-utils.test.ts +++ b/apps/desktop/src/app/chat/composer/composer-utils.test.ts @@ -2,12 +2,14 @@ import type { Unstable_TriggerItem } from '@assistant-ui/core' import { describe, expect, it } from 'vitest' import { + acceptsTriggerCompletion, isPendingDraftPersistCurrent, type PendingDraftPersist, pickPlaceholder, slashArgStage, slashChipKindForItem, - slashCommandToken + slashCommandToken, + type TriggerAcceptInput } from './composer-utils' const item = (group: string): Unstable_TriggerItem => @@ -39,6 +41,53 @@ describe('slashChipKindForItem', () => { }) }) +describe('acceptsTriggerCompletion', () => { + const press = (key: string, overrides: Partial = {}) => + acceptsTriggerCompletion({ + activeExplicit: false, + freeTextArgStage: false, + key, + kind: '/', + query: 'personality alic', + ...overrides + }) + + it('accepts on Enter / Tab / Space for a finite option list', () => { + expect(press('Enter')).toBe(true) + expect(press('Tab')).toBe(true) + expect(press(' ')).toBe(true) + }) + + it('ignores keys that are neither navigation nor acceptance', () => { + expect(press('a')).toBe(false) + expect(press('Escape')).toBe(false) + }) + + it('lets an `@` mention take a literal space', () => { + expect(press(' ', { kind: '@', query: 'src/comp' })).toBe(false) + expect(press('Enter', { kind: '@', query: 'src/comp' })).toBe(true) + }) + + it('types a space on a bare `/ ` instead of accepting', () => { + expect(press(' ', { query: '' })).toBe(false) + }) + + // The `/goal ` class: the popover may be live over free-form text, so + // the keys that mean something else in prose must keep meaning it. + it('sends the prose rather than the unchosen first row', () => { + expect(press('Enter', { freeTextArgStage: true, query: 'goal ship the redesign' })).toBe(false) + expect(press(' ', { freeTextArgStage: true, query: 'goal ship the' })).toBe(false) + }) + + it('accepts on Enter once the user has arrowed to a row deliberately', () => { + expect(press('Enter', { activeExplicit: true, freeTextArgStage: true, query: 'goal stat' })).toBe(true) + }) + + it('keeps Tab as the explicit accept even over free text', () => { + expect(press('Tab', { freeTextArgStage: true, query: 'goal stat' })).toBe(true) + }) +}) + describe('pickPlaceholder', () => { it('returns a member of the pool', () => { const pool = ['a', 'b', 'c'] as const diff --git a/apps/desktop/src/app/chat/composer/composer-utils.ts b/apps/desktop/src/app/chat/composer/composer-utils.ts index 7939be35b6b6..21e3c1ac4a12 100644 --- a/apps/desktop/src/app/chat/composer/composer-utils.ts +++ b/apps/desktop/src/app/chat/composer/composer-utils.ts @@ -4,6 +4,8 @@ import type { SlashChipKind } from '@/components/assistant-ui/directive-text' import type { ComposerAttachment } from '@/store/composer' import { setSessionPickerOpen } from '@/store/session' +import type { TriggerState } from './text-utils' + export const COMPOSER_STACK_BREAKPOINT_PX = 320 // Above the stack breakpoint but still cramped: the model pill sheds its label @@ -50,12 +52,57 @@ export function slashChipKindForItem(item: Unstable_TriggerItem): SlashChipKind return 'command' } +/** True for a skill completion — the only kind offered mid-message. */ +export const isSkillItem = (item: Unstable_TriggerItem) => slashChipKindForItem(item) === 'skill' + /** A `/` query is at its arg stage once it's past the command name. */ export const slashArgStage = (query: string) => query.includes(' ') /** The `/command` token of a slash query (`personality x` → `/personality`). */ export const slashCommandToken = (query: string) => `/${query.split(/\s+/, 1)[0]?.toLowerCase() ?? ''}` +export interface TriggerAcceptInput { + /** The user moved the highlight themselves (arrow keys) rather than + * inheriting the list's default first row. */ + activeExplicit: boolean + /** The trigger is a slash command whose argument is arbitrary prose. */ + freeTextArgStage: boolean + key: string + kind: TriggerState['kind'] + query: string +} + +/** + * Whether a keypress accepts the highlighted completion while the popover is + * open. Tab is always an accept — it has no other meaning in the composer. + * + * Enter and Space are conditional, because both mean something else while a + * free-text argument is being written (`/goal ship the redesign`). Space types + * a space, and Enter sends the message; letting either take the popover's + * pre-highlighted row would swap the prose the user is mid-sentence on for a + * subcommand they never chose. Enter still accepts once the user has arrowed + * to a row deliberately, so the highlight never lies about what Enter will do. + */ +export function acceptsTriggerCompletion({ + activeExplicit, + freeTextArgStage, + key, + kind, + query +}: TriggerAcceptInput): boolean { + if (key === 'Tab') { + return true + } + + if (key === 'Enter') { + return !freeTextArgStage || activeExplicit + } + + // Space is slash-only (an `@` mention takes a literal space) and gated to a + // non-empty query so a bare `/ ` still types a space. + return key === ' ' && kind === '/' && Boolean(query.trim()) && !freeTextArgStage +} + export interface QueueEditState { attachments: ComposerAttachment[] draft: string diff --git a/apps/desktop/src/app/chat/composer/contrib.ts b/apps/desktop/src/app/chat/composer/contrib.ts index 893f5174c200..94d19457a58f 100644 --- a/apps/desktop/src/app/chat/composer/contrib.ts +++ b/apps/desktop/src/app/chat/composer/contrib.ts @@ -3,13 +3,16 @@ * through the SAME registry schema as every other surface (statusbar, titlebar, * panes, layouts): * - * render areas (`render`): composer.top — banner strip above the input - * composer.bottom — row below the input grid - * composer.leading — inline after the "+" menu - * composer.actions — inline before the model pill + * render areas (`render`): composer.top — banner strip above the input + * composer.bottom — row below the input grid + * composer.underside — floating strip BELOW the + * whole composer (no chrome) + * composer.leading — inline after the "+" menu + * composer.actions — inline before the model pill * - * data kinds (`data`): composer.middleware (ComposerMiddleware) - * composer.attachments (ComposerAttachmentProvider) + * data kinds (`data`): composer.middleware (ComposerMiddleware) + * composer.attachments (ComposerAttachmentProvider) + * composer.microActions (ComposerMicroActionProvider) * * Core keeps ownership of the transcript, input, and submit engine — these * seams AUGMENT the composer, they never replace it. Middleware runs as an @@ -17,17 +20,23 @@ * draft, pass it through, or cancel the send by returning null. */ +import { useMemo } from 'react' + import { useContributions } from '@/contrib/react/use-contributions' import { registry } from '@/contrib/registry' +import type { TodoItem } from '@/lib/todos' import type { ComposerAttachment } from '@/store/composer' +import type { ComposerAction } from '@/store/composer-actions' export const COMPOSER_AREAS = { top: 'composer.top', bottom: 'composer.bottom', + underside: 'composer.underside', leading: 'composer.leading', actions: 'composer.actions', middleware: 'composer.middleware', - attachments: 'composer.attachments' + attachments: 'composer.attachments', + microActions: 'composer.microActions' } as const export interface ComposerDraft { @@ -92,3 +101,39 @@ export function useComposerAttachmentProviders(): Array ({ key: `${c.source ?? 'core'}:${c.id}`, ...(c.data as ComposerAttachmentProvider) })) .filter(p => Boolean(p.label && p.run)) } + +/** + * Payload of a `composer.microActions` data contribution — the pill strip at + * the top of the composer's overlay lane. + * + * `resolve` is called with the live session context and returns the badges to + * show right now, or `[]` for "nothing from me". Returning a list rather than + * a static badge is what lets a provider be conditional ("only while idle", + * "only with unfinished tasks") without a reactive `when()`, which the + * registry deliberately doesn't offer. + */ +export interface ComposerMicroActionProvider { + resolve: (ctx: ComposerMicroActionContext) => ComposerAction[] +} + +/** What a micro-action provider gets to branch on. Deliberately small: every + * field here is a standing compatibility promise to the plugins using it. */ +export interface ComposerMicroActionContext { + /** A turn is currently running in this session. */ + busy: boolean + sessionId: string + /** Live todo list for the session (empty when there is none). */ + todos: readonly TodoItem[] +} + +/** Micro-action providers, memoised against the registry's own stable + * snapshot — the strip re-resolves on every composer render, so a fresh array + * here would defeat that. */ +export function useComposerMicroActionProviders(): ComposerMicroActionProvider[] { + const contributions = useContributions(COMPOSER_AREAS.microActions) + + return useMemo( + () => contributions.map(c => c.data as ComposerMicroActionProvider).filter(p => typeof p?.resolve === 'function'), + [contributions] + ) +} diff --git a/apps/desktop/src/app/chat/composer/controls.test.tsx b/apps/desktop/src/app/chat/composer/controls.test.tsx index 90d38d274954..8a84be8ea477 100644 --- a/apps/desktop/src/app/chat/composer/controls.test.tsx +++ b/apps/desktop/src/app/chat/composer/controls.test.tsx @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { ChatBarState } from '@/app/chat/composer/types' import { I18nProvider } from '@/i18n' +import { applyWakeStartResult, applyWakeStatus, resetWakeWordState } from '@/store/wake-word' import { ComposerControls } from './controls' @@ -77,3 +78,62 @@ describe('ComposerControls shortcut tooltips', () => { await expectShortcutTooltip('Queue message', 'Ctrl+↵') }) }) + +describe('wake-word ear visibility', () => { + afterEach(() => { + resetWakeWordState() + }) + + it('stays mounted during a busy agent turn', () => { + applyWakeStatus({ available: true, enabled: true, listening: true, phrase: 'hey hermes' }) + renderControls({ busy: true, busyAction: 'stop' }) + + expect(screen.getByLabelText('Wake word: "hey hermes" — listening')).toBeTruthy() + }) + + it('stays mounted (enabled in config) even when a start was refused', () => { + applyWakeStatus({ available: true, enabled: true, listening: false, phrase: 'hey hermes' }) + // Transient refusal marks available false but enabled keeps it mounted. + applyWakeStartResult({ hint: 'mic busy', reason: 'unavailable', started: false }) + renderControls() + + expect(screen.getByLabelText('Wake word: "hey hermes" — off')).toBeTruthy() + }) + + it('stays visible (never hides) even when unavailable and not enabled', () => { + applyWakeStatus({ available: false, enabled: false, listening: false, phrase: 'hey hermes' }) + renderControls() + + // The ear ALWAYS shows so the user can click to enable; a failed start + // surfaces its reason in the tooltip rather than hiding the control. + expect(screen.getByLabelText('Wake word: "hey hermes" — off')).toBeTruthy() + }) + + it('surfaces the backend refusal reason in the tooltip, still visible', () => { + applyWakeStatus({ available: false, enabled: false, listening: false, phrase: 'hey hermes' }) + applyWakeStartResult({ hint: 'run `hermes tools` (Voice section)', reason: 'unavailable', started: false }) + renderControls() + + const ear = screen.getByLabelText('Wake word: "hey hermes" — off') + expect(ear).toBeTruthy() + }) + + it('shows a disabled paused ear inside the voice-conversation pill', () => { + applyWakeStatus({ available: true, enabled: true, listening: true, phrase: 'hey hermes' }) + renderControls({ + conversation: { + active: true, + level: 0, + muted: false, + onEnd: vi.fn(), + onStart: vi.fn(), + onStopTurn: vi.fn(), + onToggleMute: vi.fn(), + status: 'listening' + } + }) + + const ear = screen.getByLabelText('Wake word: "hey hermes" — paused during voice chat') + expect((ear as HTMLButtonElement).disabled).toBe(true) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/controls.tsx b/apps/desktop/src/app/chat/composer/controls.tsx index 996b962f5448..06a3869894fe 100644 --- a/apps/desktop/src/app/chat/composer/controls.tsx +++ b/apps/desktop/src/app/chat/composer/controls.tsx @@ -1,10 +1,24 @@ +import { useStore } from '@nanostores/react' + import { Button } from '@/components/ui/button' import { Codicon } from '@/components/ui/codicon' import { Tip, TipKeybindLabel } from '@/components/ui/tooltip' import { useI18n } from '@/i18n' import { triggerHaptic } from '@/lib/haptics' -import { AudioLines, iconSize, Layers3, Loader2, Square, SteeringWheel, Volume2, VolumeX } from '@/lib/icons' +import { + AudioLines, + Ear, + EarOff, + iconSize, + Layers3, + Loader2, + Square, + SteeringWheel, + Volume2, + VolumeX +} from '@/lib/icons' import { cn } from '@/lib/utils' +import { $wakeWord, toggleWakeWord } from '@/store/wake-word' import type { ConversationStatus } from './hooks/use-voice-conversation' import { ModelPill } from './model-pill' @@ -80,6 +94,7 @@ export function ComposerControls({ + {busyAction === 'steer' ? ( }> + + ) +} + function DictationButton({ disabled, state, diff --git a/apps/desktop/src/app/chat/composer/directive-actions.test.tsx b/apps/desktop/src/app/chat/composer/directive-actions.test.tsx new file mode 100644 index 000000000000..4c827fe9a164 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/directive-actions.test.tsx @@ -0,0 +1,150 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { I18nProvider } from '@/i18n' + +import { ComposerDirectiveActions } from './directive-actions' +import { refChipElement } from './rich-editor' + +const desktopWindow = window as unknown as { hermesDesktop?: Window['hermesDesktop'] } + +const openSession = vi.fn() + +vi.mock('@/app/open-session', () => ({ openSession: (...args: unknown[]) => openSession(...args) })) + +/** A live contenteditable holding real chips, with the watcher bound to it — + * the same pair both composers mount. */ +function mountEditor(chips: { kind: string; value: string }[]) { + const editor = document.createElement('div') + + editor.contentEditable = 'true' + editor.append(...chips.map(chip => refChipElement(chip.kind, `\`${chip.value}\``))) + document.body.append(editor) + + render( + + + + ) + + return editor +} + +function chips(editor: HTMLElement, kind: string) { + return Array.from(editor.querySelectorAll(`[data-ref-kind="${kind}"]`)) +} + +function hover(node: Element) { + fireEvent.pointerOver(node, { bubbles: true }) +} + +/** The reference the visible action pill points at, or null when there is none. */ +function pillValue() { + return document.querySelector('[data-slot="composer-directive-action"]')?.getAttribute('data-value') ?? null +} + +afterEach(() => { + cleanup() + document.body.replaceChildren() + delete desktopWindow.hermesDesktop + openSession.mockReset() + vi.useRealTimers() +}) + +describe('ComposerDirectiveActions', () => { + it('offers an action for a hovered actionable chip', () => { + const editor = mountEditor([{ kind: 'url', value: 'https://example.com/docs' }]) + + expect(pillValue()).toBeNull() + + hover(chips(editor, 'url')[0]!) + + expect(pillValue()).toBe('https://example.com/docs') + }) + + it('opens a url externally rather than navigating the app', () => { + const openExternal = vi.fn().mockResolvedValue(undefined) + + desktopWindow.hermesDesktop = { openExternal } as unknown as Window['hermesDesktop'] + + const editor = mountEditor([{ kind: 'url', value: 'https://example.com/docs' }]) + + hover(chips(editor, 'url')[0]!) + fireEvent.click(screen.getByRole('button')) + + expect(openExternal).toHaveBeenCalledWith('https://example.com/docs') + expect(pillValue()).toBeNull() + }) + + it('runs the kind-specific action — a session chip opens the session', async () => { + const editor = mountEditor([{ kind: 'session', value: 'default/20260722_204335_d62c16' }]) + + hover(chips(editor, 'session')[0]!) + fireEvent.click(screen.getByRole('button')) + // openSessionRef lazy-imports the navigator, so the call lands a tick later. + await vi.waitFor(() => + expect(openSession).toHaveBeenCalledWith('20260722_204335_d62c16', expect.any(Function), 'tab') + ) + }) + + it('leaves kinds with no action alone', () => { + const editor = mountEditor([{ kind: 'file', value: 'src/main.tsx' }]) + + hover(chips(editor, 'file')[0]!) + + expect(pillValue()).toBeNull() + }) + + it('follows the pointer from one chip to the next', () => { + const editor = mountEditor([ + { kind: 'url', value: 'https://one.example' }, + { kind: 'url', value: 'https://two.example' } + ]) + + const [first, second] = chips(editor, 'url') + + hover(first!) + + expect(pillValue()).toBe('https://one.example') + + hover(second!) + + expect(pillValue()).toBe('https://two.example') + }) + + it('keeps the pill up while the pointer crosses onto it', () => { + vi.useFakeTimers() + + const editor = mountEditor([{ kind: 'url', value: 'https://example.com' }]) + const chip = chips(editor, 'url')[0]! + + hover(chip) + fireEvent.pointerOut(chip, { relatedTarget: document.body }) + fireEvent.mouseEnter(screen.getByRole('button').parentElement!) + vi.advanceTimersByTime(500) + + expect(pillValue()).toBe('https://example.com') + }) + + it('binds to the document so a late-attached editor still gets the affordance', () => { + // The edit composer's editor isn't reliably in the DOM when the effect + // first runs; a document listener that reads the editor lazily works + // regardless — this is the whole reason it binds to document, not editor. + const editor = document.createElement('div') + + editor.contentEditable = 'true' + editor.append(refChipElement('url', '`https://late.example`')) + + render( + + + + ) + + // Editor attached AFTER mount. + document.body.append(editor) + hover(editor.querySelector('[data-ref-kind="url"]')!) + + expect(pillValue()).toBe('https://late.example') + }) +}) diff --git a/apps/desktop/src/app/chat/composer/directive-actions.tsx b/apps/desktop/src/app/chat/composer/directive-actions.tsx new file mode 100644 index 000000000000..b18ade2aac03 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/directive-actions.tsx @@ -0,0 +1,154 @@ +/** + * Hover actions for directive chips in a composer. + * + * A directive chip (`@url:`, `@session:`, …) reads as the thing it points at + * and is coloured like one, but a composer is an editor — a click inside the + * contenteditable only places the caret, so there's no way to *act* on the + * reference. Instead, hovering a chip whose kind has an action floats a small + * pill above it that runs it. + * + * The kind → action table (`DIRECTIVE_ACTIONS`) lives in `directive-text`, so + * it is shared with the sent-message chip: one entry lights up both surfaces. + */ +import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' +import { createPortal } from 'react-dom' + +import { DIRECTIVE_ACTIONS, type DirectiveAction } from '@/components/assistant-ui/directive-text' +import { composerFloatingPill } from '@/components/chat/composer-dock' +import { Codicon } from '@/components/ui/codicon' +import { useI18n } from '@/i18n' +import { cn } from '@/lib/utils' + +/** Moving between the chip and the pill crosses a gap where neither is hovered. + * Short enough that it still reads as instant on the way out. */ +const HIDE_DELAY_MS = 120 + +/** The actionable directive chip under `target` that also belongs to `editor`, + * if there is one. */ +function actionableChipAt(target: EventTarget | null, editor: HTMLElement): HTMLElement | null { + const chip = target instanceof Element ? target.closest('[data-ref-kind]') : null + const kind = chip?.dataset.refKind + + return chip && kind && chip.dataset.refId && editor.contains(chip) && DIRECTIVE_ACTIONS[kind] ? chip : null +} + +interface Anchor { + action: DirectiveAction + chip: HTMLElement + left: number + top: number + value: string +} + +function anchorFor(chip: HTMLElement): Anchor | null { + const value = chip.dataset.refId + const action = chip.dataset.refKind ? DIRECTIVE_ACTIONS[chip.dataset.refKind] : undefined + + if (!value || !action || !chip.isConnected) { + return null + } + + const rect = chip.getBoundingClientRect() + + return { action, chip, left: rect.left, top: rect.top, value } +} + +/** + * Renders the action pill for whichever actionable chip in `editorRef` is + * hovered. + * + * Listeners bind to `document`, not the editor, so mount timing can't strand + * them: the edit composer's contenteditable isn't reliably attached when this + * effect first runs, and a document listener that reads the editor lazily works + * regardless. Each instance filters to its own editor, so the docked and edit + * composers never show two pills for one chip. + */ +export function ComposerDirectiveActions({ editorRef }: { editorRef: RefObject }) { + const { t } = useI18n() + const [anchor, setAnchor] = useState(null) + const hideTimerRef = useRef(undefined) + + const cancelHide = useCallback(() => { + window.clearTimeout(hideTimerRef.current) + }, []) + + const hideSoon = useCallback(() => { + cancelHide() + hideTimerRef.current = window.setTimeout(() => setAnchor(null), HIDE_DELAY_MS) + }, [cancelHide]) + + useEffect(() => { + const onPointerOver = (event: PointerEvent) => { + const editor = editorRef.current + const chip = editor && actionableChipAt(event.target, editor) + + if (!chip) { + return + } + + cancelHide() + setAnchor(current => (current?.chip === chip ? current : anchorFor(chip))) + } + + const onPointerOut = (event: PointerEvent) => { + const editor = editorRef.current + const chip = editor && actionableChipAt(event.target, editor) + + // A move within the same chip (its icon → its label) is not a leave. + if (chip && editor && chip === actionableChipAt(event.relatedTarget, editor)) { + return + } + + hideSoon() + } + + // The chip can move or vanish under a parked pointer: the editor scrolls, + // the window resizes, or the user deletes the reference the pill points at. + const reanchor = () => setAnchor(current => (current ? anchorFor(current.chip) : null)) + + document.addEventListener('pointerover', onPointerOver) + document.addEventListener('pointerout', onPointerOut) + window.addEventListener('scroll', reanchor, true) + window.addEventListener('resize', reanchor) + + return () => { + document.removeEventListener('pointerover', onPointerOver) + document.removeEventListener('pointerout', onPointerOut) + window.removeEventListener('scroll', reanchor, true) + window.removeEventListener('resize', reanchor) + window.clearTimeout(hideTimerRef.current) + } + }, [cancelHide, editorRef, hideSoon]) + + if (!anchor) { + return null + } + + return createPortal( +
+ +
, + document.body + ) +} diff --git a/apps/desktop/src/app/chat/composer/directive-label.test.ts b/apps/desktop/src/app/chat/composer/directive-label.test.ts new file mode 100644 index 000000000000..f6efe18b42f1 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/directive-label.test.ts @@ -0,0 +1,131 @@ +import type { Unstable_TriggerItem } from '@assistant-ui/core' +import { act, renderHook } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' + +import { hermesDirectiveFormatter } from '@/components/assistant-ui/directive-text' + +import { classify } from './hooks/use-at-completions' +import { useComposerTrigger } from './hooks/use-composer-trigger' +import { composerPlainText, RICH_INPUT_SLOT } from './rich-editor' + +/** A row exactly as tui_gateway's complete.path emits it, run through the + * real classify() the popover uses. */ +function backendRow(text: string, display: string, meta: string): Unstable_TriggerItem { + const c = classify({ text, display, meta }) + + return { + id: `${text}|0`, + type: c.type, + label: c.display, + metadata: { icon: c.type, display: c.display, meta: c.meta, rawText: text, insertId: c.insertId } + } +} + +function typed(text: string) { + const editor = document.createElement('div') + + editor.contentEditable = 'true' + editor.dataset.slot = RICH_INPUT_SLOT + document.body.append(editor) + editor.append(document.createTextNode(text)) + + const range = document.createRange() + + range.selectNodeContents(editor) + range.collapse(false) + + const sel = window.getSelection() + + sel?.removeAllRanges() + sel?.addRange(range) + + const editorRef = { current: editor as HTMLDivElement | null } + + const { result } = renderHook(() => + useComposerTrigger({ + at: { adapter: null, loading: false }, + draftRef: { current: text }, + editorRef, + requestMainFocus: vi.fn(), + setComposerText: vi.fn(), + slash: { adapter: null, loading: false } + }) + ) + + act(() => result.current.refreshTrigger()) + + return { editor, result } +} + +/** The label the sent message renders for a committed draft. */ +function sentLabel(draft: string) { + return hermesDirectiveFormatter + .parse(draft) + .filter((s): s is Extract => s.kind === 'mention') + .map(s => s.label) + .join(',') +} + +describe('one label per reference, on every surface', () => { + it('the popover row, the committed chip, and the sent chip all read the same', () => { + const cases = [ + { text: '@folder:apps/desktop/', display: 'desktop/', meta: 'dir' }, + { text: '@file:apps/desktop/src/main.tsx', display: 'main.tsx', meta: 'apps/desktop/src' }, + { text: '@folder:apps/desktop/src/', display: 'src/', meta: 'dir' } + ] + + for (const entry of cases) { + const item = backendRow(entry.text, entry.display, entry.meta) + const { editor, result } = typed('@desk') + + act(() => result.current.replaceTriggerWithChip(item)) + + const row = String((item.metadata as { display: string }).display) + const chip = editor.querySelector('[data-ref-text]')?.textContent ?? '' + + expect(chip).toBe(row) + expect(sentLabel(composerPlainText(editor))).toBe(row) + } + }) + + it('a folder pick reads as its path, not a bare basename', () => { + // `src` and `desktop` repeat all over a repo — the row you picked said + // where it was, and the chip has to keep saying it. + const item = backendRow('@folder:apps/desktop/', 'desktop/', 'dir') + + expect(item.label).toBe('apps/desktop/') + + const { editor, result } = typed('@desk') + + act(() => result.current.replaceTriggerWithChip(item)) + + expect(editor.querySelector('[data-ref-text]')?.textContent).toBe('apps/desktop/') + }) + + it('Tab-descend leaves the live query, and the scope when there is one', () => { + const { editor, result } = typed('@folder:desk') + + act(() => + result.current.replaceTriggerWithChip(backendRow('@folder:apps/desktop/', 'desktop/', 'dir'), { + descend: true + }) + ) + + // Mid-browse the editor holds the live query, scope included — that's the + // path being typed, not a label, and it's what the next completion reads. + expect(composerPlainText(editor)).toBe('@folder:apps/desktop/') + }) + + it('a url still reads host + path on every surface', () => { + const item = backendRow('@url:https://github.com/NousResearch/hermes-agent/pull/74533', '', '') + const { editor, result } = typed('@gith') + + act(() => result.current.replaceTriggerWithChip(item)) + + const expected = 'github.com/NousResearch/hermes-agent/pull/74533' + + expect(item.label).toBe(expected) + expect(editor.querySelector('[data-ref-text]')?.textContent).toBe(expected) + expect(sentLabel(composerPlainText(editor))).toBe(expected) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/directive-scope.test.ts b/apps/desktop/src/app/chat/composer/directive-scope.test.ts new file mode 100644 index 000000000000..08d1804dfbab --- /dev/null +++ b/apps/desktop/src/app/chat/composer/directive-scope.test.ts @@ -0,0 +1,164 @@ +import type { Unstable_TriggerItem } from '@assistant-ui/core' +import { act, renderHook } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' + +import { useComposerTrigger } from './hooks/use-composer-trigger' +import { pathifyRefs } from './path-refs' +import { composerPlainText, insertComposerContentsAtCaret, RICH_INPUT_SLOT } from './rich-editor' +import { detectTrigger, openDirectiveScope, textBeforeCaret } from './text-utils' +import { linkifyUrls } from './url-refs' + +function folderItem(rel: string): Unstable_TriggerItem { + const rawText = `@folder:${rel}/` + + return { + id: `${rawText}|0`, + type: 'folder', + label: rel.split('/').filter(Boolean).pop() ?? rel, + metadata: { icon: 'folder', display: `${rel}/`, meta: 'dir', rawText, insertId: `${rel}/` } + } +} + +/** Literally-typed text, caret `fromEnd` characters before the end. */ +function typed(text: string, fromEnd = 0) { + const editor = document.createElement('div') + + editor.contentEditable = 'true' + editor.dataset.slot = RICH_INPUT_SLOT + document.body.append(editor) + + const node = document.createTextNode(text) + + editor.append(node) + + const range = document.createRange() + + range.setStart(node, text.length - fromEnd) + range.collapse(true) + + const sel = window.getSelection() + + sel?.removeAllRanges() + sel?.addRange(range) + + return editor +} + +function withTrigger(editor: HTMLDivElement, draft: string) { + const editorRef = { current: editor as HTMLDivElement | null } + + const { result } = renderHook(() => + useComposerTrigger({ + at: { adapter: null, loading: false }, + draftRef: { current: draft }, + editorRef, + requestMainFocus: vi.fn(), + setComposerText: vi.fn(), + slash: { adapter: null, loading: false } + }) + ) + + act(() => result.current.refreshTrigger()) + + return result +} + +/** The composer's paste handler, minus the clipboard plumbing. */ +function paste(editor: HTMLDivElement, text: string) { + insertComposerContentsAtCaret(editor, pathifyRefs(linkifyUrls(text)), openDirectiveScope(editor)) +} + +describe('directive scope is a browse mode, not text to maintain', () => { + it('Tab-descend carries the scope down instead of dropping to a bare path', () => { + const editor = typed('@folder:apps/deskt') + const result = withTrigger(editor, '@folder:apps/deskt') + + expect(result.current.trigger).toMatchObject({ kind: '@', scope: 'folder', value: 'apps/deskt' }) + + act(() => result.current.replaceTriggerWithChip(folderItem('apps/desktop'), { descend: true })) + + expect(composerPlainText(editor)).toBe('@folder:apps/desktop/') + }) + + it('Backspace climbs the path, then drops the whole scope', () => { + const editor = typed('@folder:apps/desktop/') + const result = withTrigger(editor, '@folder:apps/desktop/') + + act(() => result.current.ascendTriggerPath()) + expect(composerPlainText(editor)).toBe('@folder:apps/') + + act(() => result.current.refreshTrigger()) + act(() => result.current.ascendTriggerPath()) + expect(composerPlainText(editor)).toBe('@folder:') + + // The scope is one unit: Backspace drops it whole rather than nibbling + // back through `:`, `r`, `e`, `d`, `l`, `o`, `f`. + act(() => result.current.refreshTrigger()) + act(() => result.current.ascendTriggerPath()) + expect(composerPlainText(editor)).toBe('@') + }) + + it('leaves Backspace alone when there is no scope and no path', () => { + const editor = typed('@apps') + const result = withTrigger(editor, '@apps') + + let handled = true + + act(() => { + handled = result.current.ascendTriggerPath() + }) + + expect(handled).toBe(false) + }) + + it('a pick mid-message keeps the trailing prose and consumes the whole token', () => { + const editor = typed('@folder:apps/deskt and some trailing words', 24) + const result = withTrigger(editor, '@folder:apps/deskt and some trailing words') + + act(() => result.current.replaceTriggerWithChip(folderItem('apps/desktop'))) + + expect(composerPlainText(editor)).toBe('@folder:`apps/desktop/` and some trailing words') + expect(editor.querySelector('[data-ref-kind="folder"]')).not.toBeNull() + }) + + it('pasting into an open @url: scope consumes it instead of stacking', () => { + const editor = typed('refer to @url:') + + paste(editor, 'https://github.com/NousResearch/hermes-agent/pull/74533') + + expect(composerPlainText(editor)).toBe('refer to @url:`https://github.com/NousResearch/hermes-agent/pull/74533`') + expect(editor.textContent).not.toContain('@url:@url:') + }) + + it('a normal paste with no open scope is untouched', () => { + const editor = typed('look at ') + + paste(editor, 'https://example.com/x') + + expect(composerPlainText(editor)).toBe('look at @url:`https://example.com/x`') + }) + + it('scope parsing leaves an unscoped @ query alone', () => { + expect(detectTrigger('@apps/desk')).toMatchObject({ kind: '@', value: 'apps/desk' }) + expect(detectTrigger('@apps/desk')?.scope).toBeUndefined() + }) + + it('openDirectiveScope only fires on an EMPTY scope', () => { + // The count is what a paste consumes: `@url:` is 5 characters of syntax + // the user never typed and shouldn't be left holding. + expect(openDirectiveScope(typed('@url:'))).toBe(5) + expect(openDirectiveScope(typed('@url:https://x.com'))).toBe(0) + expect(openDirectiveScope(typed('plain text'))).toBe(0) + }) + + it('chips stay atomic to scope detection', () => { + const editor = typed('@folder:apps/desktop/') + const result = withTrigger(editor, '@folder:apps/desktop/') + + act(() => result.current.replaceTriggerWithChip(folderItem('apps/desktop'))) + + // A committed chip is one object-replacement char, so a fresh `@` typed + // after it opens an unscoped browse rather than inheriting the old scope. + expect(detectTrigger(`${textBeforeCaret(editor)}@`)?.scope).toBeUndefined() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/empty-composer.test.ts b/apps/desktop/src/app/chat/composer/empty-composer.test.ts new file mode 100644 index 000000000000..dcf3398e6bcb --- /dev/null +++ b/apps/desktop/src/app/chat/composer/empty-composer.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from 'vitest' + +import { + beginComposerComposition, + composerPlainText, + deleteChipBeforeCaret, + normalizeComposerEditorDom, + renderComposerContents, + RICH_INPUT_SLOT +} from './rich-editor' + +function editor(): HTMLDivElement { + const el = document.createElement('div') + + el.dataset.slot = RICH_INPUT_SLOT + el.contentEditable = 'true' + document.body.append(el) + + return el +} + +/** Whatever emptied it — Delete, cut, Chromium's own selection-delete — the + * normalizer lands on the same DOM. */ +function emptied(): HTMLDivElement { + const el = editor() + + el.append(document.createTextNode('hello')) + el.replaceChildren() + normalizeComposerEditorDom(el) + + return el +} + +describe('an emptied composer reads as empty', () => { + it('keeps the placeholder
so the contenteditable holds its height', () => { + // The scaffolding is deliberate: a childless contenteditable collapses to a + // sliver in Chromium. It just must not read as content. + expect(emptied().innerHTML).toBe('
') + }) + + it('reads that editor as empty, not as a newline', () => { + expect(composerPlainText(emptied())).toBe('') + }) + + it('reads a truly childless editor as empty', () => { + expect(composerPlainText(editor())).toBe('') + }) + + it('still reads a real Shift+Enter line break as a newline', () => { + const el = editor() + + el.append(document.createTextNode('one'), document.createElement('br'), document.createTextNode('two')) + + expect(composerPlainText(el)).toBe('one\ntwo') + }) + + it('still reads a trailing break after text as a newline', () => { + const el = editor() + + el.append(document.createTextNode('one'), document.createElement('br')) + + expect(composerPlainText(el)).toBe('one\n') + }) + + it('only treats the EDITOR\u2019s lone
as scaffolding, not a nested one', () => { + // A lone
inside some other element is a real line break; the exemption + // is scoped to the editor root by its slot marker. (The block wrapper adds + // its own trailing newline — unchanged behavior, asserted so the exemption + // can't quietly widen to nested nodes.) + const el = editor() + const inner = document.createElement('div') + + inner.append(document.createElement('br')) + el.append(document.createTextNode('one'), inner) + + expect(composerPlainText(el)).toBe('one\n\n') + }) +}) + +/** The rule the stylesheet paints the placeholder with. `:empty` alone goes + * false the instant the scaffolding
lands. */ +const PLACEHOLDER_SHOWS = ':is(:empty, [data-empty])' + +describe('an emptied composer shows its placeholder again', () => { + it('advertises emptiness once the scaffolding break is in place', () => { + expect(emptied().matches(PLACEHOLDER_SHOWS)).toBe(true) + }) + + it('advertises emptiness for a truly childless editor', () => { + expect(editor().matches(PLACEHOLDER_SHOWS)).toBe(true) + }) + + it('stops advertising it once something is typed', () => { + const el = emptied() + + el.replaceChildren(document.createTextNode('hi')) + normalizeComposerEditorDom(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(false) + }) + + // A text node is invisible to selectors, so `one
` and `
` are the same + // shape to any pure-CSS rule (`:has(> br:only-child)` matches both and paints + // the placeholder straight over the user's text). The DOM writer has to say. + it('does not advertise emptiness for a trailing break after text', () => { + const el = editor() + + el.append(document.createTextNode('one'), document.createElement('br')) + normalizeComposerEditorDom(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(false) + }) + + it('does not advertise emptiness for a Shift+Enter break between text', () => { + const el = editor() + + el.append(document.createTextNode('one'), document.createElement('br'), document.createTextNode('two')) + normalizeComposerEditorDom(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(false) + }) + + // Repainting from text (restored draft, undo, completion rebuild) is the + // other writer that reshapes the editor root — it must not strand the marker. + it('drops the marker when a draft is painted back in', () => { + const el = emptied() + + renderComposerContents(el, 'restored draft') + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(false) + }) + + it('re-advertises emptiness when a draft is painted back out', () => { + const el = editor() + + renderComposerContents(el, 'temporary') + renderComposerContents(el, '') + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(true) + }) + + // Chromium leaves zero-length text nodes behind whenever an edit lands next + // to a contenteditable=false chip. They render as nothing, so an editor + // holding only those is empty to the user — counting them as contents left + // the placeholder hidden under a composer that looked blank. + it('advertises emptiness for an editor holding only zero-length text nodes', () => { + const el = editor() + + el.append(document.createTextNode(''), document.createTextNode('')) + normalizeComposerEditorDom(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(true) + }) + + it('does not advertise emptiness while real text sits beside that litter', () => { + const el = editor() + + el.append(document.createTextNode(''), document.createTextNode('one'), document.createTextNode('')) + normalizeComposerEditorDom(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(false) + }) + + // Input events are skipped for the duration of an IME composition, so nothing + // else clears the marker until it ends — the hint would sit behind the + // hiragana the user is composing (#75960). + it('hides the placeholder before IME preedit text starts', () => { + const el = emptied() + + beginComposerComposition(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(false) + }) + + it('brings the placeholder back when composition ends with nothing committed', () => { + const el = emptied() + + beginComposerComposition(el) + normalizeComposerEditorDom(el) + + expect(el.matches(PLACEHOLDER_SHOWS)).toBe(true) + }) +}) + +/** A directive chip, as `refChipElement` builds it. */ +function chip(): HTMLSpanElement { + const el = document.createElement('span') + + el.contentEditable = 'false' + el.dataset.refText = '@folder:`apps/desktop/`' + el.append(document.createTextNode('apps/desktop/')) + + return el +} + +function caretAt(node: Node, offset: number) { + const range = document.createRange() + + range.setStart(node, offset) + range.collapse(true) + + const selection = window.getSelection() + + selection?.removeAllRanges() + selection?.addRange(range) +} + +/** Committing a completion empties the typed token's text node instead of + * removing it, and `Range.insertNode` splits the line around the caret — so a + * freshly-chipped directive sits between zero-length text nodes. Backspace has + * to see past them or the chip can't be deleted at all. */ +describe('backspace deletes a chip surrounded by Chromium litter', () => { + it('deletes the chip when the caret sits in a zero-length text node after it', () => { + const el = editor() + + el.append(document.createTextNode(''), chip(), document.createTextNode('')) + caretAt(el.childNodes[2] as Node, 0) + + expect(deleteChipBeforeCaret(el)).toBe(true) + expect(el.querySelector('[data-ref-text]')).toBeNull() + }) + + it('deletes the chip when the caret is past a zero-length text node at editor level', () => { + const el = editor() + + el.append(chip(), document.createTextNode('')) + caretAt(el, 2) + + expect(deleteChipBeforeCaret(el)).toBe(true) + expect(el.querySelector('[data-ref-text]')).toBeNull() + }) + + it('still swallows the auto-inserted trailing space through that litter', () => { + const el = editor() + + el.append(chip(), document.createTextNode(''), document.createTextNode(' ')) + caretAt(el, 2) + + expect(deleteChipBeforeCaret(el)).toBe(true) + expect(composerPlainText(el)).toBe('') + }) + + it('keeps real following text when it deletes the chip', () => { + const el = editor() + + el.append(chip(), document.createTextNode(''), document.createTextNode(' and this')) + caretAt(el, 2) + + expect(deleteChipBeforeCaret(el)).toBe(true) + expect(composerPlainText(el)).toBe('and this') + }) + + it('leaves plain text to the native backspace', () => { + const el = editor() + + el.append(document.createTextNode('hello')) + caretAt(el.firstChild as Node, 5) + + expect(deleteChipBeforeCaret(el)).toBe(false) + }) + + it('sweeps the litter out of the editor when it normalizes', () => { + const el = editor() + + el.append(document.createTextNode(''), chip(), document.createTextNode('')) + normalizeComposerEditorDom(el) + + expect(Array.from(el.childNodes).map(node => node.nodeName)).toEqual(['SPAN']) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/enter-submit-dom-race.test.tsx b/apps/desktop/src/app/chat/composer/enter-submit-dom-race.test.tsx index ff01bf6fd377..ad35b20faa33 100644 --- a/apps/desktop/src/app/chat/composer/enter-submit-dom-race.test.tsx +++ b/apps/desktop/src/app/chat/composer/enter-submit-dom-race.test.tsx @@ -29,7 +29,8 @@ function Harness({ onSubmit, onQueue, onCancel, - onDrain + onDrain, + onSendNow }: { busy?: boolean disabled?: boolean @@ -38,6 +39,7 @@ function Harness({ onQueue: (text: string) => void onCancel: () => void onDrain: () => void + onSendNow?: (id: string) => void }) { const editorRef = useRef(null) const draftRef = useRef('') @@ -103,6 +105,12 @@ function Harness({ } if (busy && !hasLivePayload) { + const head = queued[0] + + if (head) { + onSendNow?.(head) + } + return } @@ -167,13 +175,14 @@ describe('composer Enter submit — live DOM vs stale composer state (#39630)', expect(onCancel).not.toHaveBeenCalled() }) - it('treats an empty Enter while busy as a no-op (never an accidental Stop)', async () => { + it('treats an empty Enter while busy with nothing queued as a no-op (never an accidental Stop)', async () => { const onCancel = vi.fn() const onSubmit = vi.fn() const onQueue = vi.fn() + const onSendNow = vi.fn() const { getByTestId } = render( - + ) const editor = getByTestId('editor') @@ -186,6 +195,35 @@ describe('composer Enter submit — live DOM vs stale composer state (#39630)', expect(onCancel).not.toHaveBeenCalled() expect(onSubmit).not.toHaveBeenCalled() expect(onQueue).not.toHaveBeenCalled() + expect(onSendNow).not.toHaveBeenCalled() + }) + + it('double-send: an empty Enter while busy with a queued turn sends that turn now', async () => { + const onCancel = vi.fn() + const onSendNow = vi.fn() + + const { getByTestId } = render( + + ) + + const editor = getByTestId('editor') + + await act(async () => { + editor.textContent = '' + fireEvent.keyDown(editor, { key: 'Enter' }) + }) + + // Head of the queue, and NOT a bare cancel — send-now promotes + interrupts. + expect(onSendNow).toHaveBeenCalledWith('queued-1') + expect(onCancel).not.toHaveBeenCalled() }) it('drains the next queued prompt on Enter when idle with a truly empty editor', async () => { diff --git a/apps/desktop/src/app/chat/composer/focus.test.ts b/apps/desktop/src/app/chat/composer/focus.test.ts new file mode 100644 index 000000000000..4fd210873361 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/focus.test.ts @@ -0,0 +1,285 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { $hoveredTreeGroup } from '@/components/pane-shell/tree/store' + +import { + blurComposerInput, + getActiveComposer, + markActiveComposer, + onComposerFocusRequest, + onComposerModelMenuRequest, + releaseActiveComposer, + requestComposerFocus, + requestModelMenuToggle +} from './focus' +import { RICH_INPUT_SLOT } from './rich-editor' + +/** + * Inactive tabs keep their composer mounted, so an unscoped lookup can blur a + * background input and leave the one the user is typing in focused. + */ + +/** A composer input inside its own pane layer, hidden or not. */ +function mountInput(hidden = false) { + const layer = document.createElement('div') + const input = document.createElement('div') + input.dataset.slot = RICH_INPUT_SLOT + input.tabIndex = 0 + layer.toggleAttribute('data-pane-hidden', hidden) + layer.append(input) + document.body.append(layer) + + return input +} + +/** A chat surface stamp — the same `data-composer-target` ChatView hangs. */ +function mountSurface(target: string, hidden = false) { + const layer = document.createElement('div') + layer.toggleAttribute('data-pane-hidden', hidden) + const surface = document.createElement('div') + surface.dataset.composerTarget = target + layer.append(surface) + document.body.append(layer) + + return surface +} + +afterEach(() => { + document.body.innerHTML = '' + // `activeTarget` is module-level — a case that leaves a stale claim behind + // would otherwise decide the next one. + markActiveComposer('main') + $hoveredTreeGroup.set(null) +}) + +describe('blurComposerInput', () => { + it('blurs the foreground composer while a hidden tab matches first', () => { + const background = mountInput(true) + const foreground = mountInput() + + foreground.focus() + blurComposerInput() + + expect(document.activeElement).not.toBe(foreground) + expect(document.activeElement).not.toBe(background) + }) + + it('leaves focus alone when the composer does not hold it', () => { + const outside = document.createElement('button') + document.body.append(outside) + mountInput() + + outside.focus() + blurComposerInput() + + expect(document.activeElement).toBe(outside) + }) +}) + +/** + * `markActiveComposer` has four call sites and, unguarded, no counterpart: an + * unmounting or keep-alive-buried composer left `activeTarget` pointing at + * itself, so every `'active'`-routed request was delivered to a target with no + * on-screen subscriber. Type-to-focus preventDefaults the keystroke BEFORE the + * request, so a dead target swallows the character and focuses nothing. + */ +describe('releaseActiveComposer', () => { + it('falls back to the main composer when the claimant releases', () => { + const root = document.createElement('div') + root.dataset.slot = 'aui_edit-composer-root' + document.body.append(root) + + markActiveComposer('edit') + expect(getActiveComposer()).toBe('edit') + + root.remove() + releaseActiveComposer('edit') + + expect(getActiveComposer()).toBe('main') + }) + + it('leaves the key with the live claimant when a stale composer releases late', () => { + markActiveComposer('edit') + markActiveComposer('tile:abc') + + releaseActiveComposer('edit') + + expect(getActiveComposer()).toBe('tile:abc') + }) + + it('prefers the visible chat surface over a hard main default', () => { + const root = document.createElement('div') + root.dataset.slot = 'aui_edit-composer-root' + document.body.append(root) + mountSurface('tile:visible') + markActiveComposer('edit') + + root.remove() + releaseActiveComposer('edit') + + expect(getActiveComposer()).toBe('tile:visible') + }) + + it('routes an active-target request to the main composer once the edit composer closes', async () => { + // Mirrors the per-composer filter in use-composer-draft / user-edit-composer: + // a composer ignores any request not addressed to its own target. + const mainComposerSaw: string[] = [] + + const off = onComposerFocusRequest(({ target }) => { + if (target === 'main') { + mainComposerSaw.push(target) + } + }) + + const root = document.createElement('div') + root.dataset.slot = 'aui_edit-composer-root' + document.body.append(root) + markActiveComposer('edit') + root.remove() + releaseActiveComposer('edit') + requestComposerFocus('active') + + // `dispatch` defers to a macrotask so click/keydown handlers settle first. + await new Promise(resolve => window.setTimeout(resolve, 0)) + off() + + expect(mainComposerSaw).toEqual(['main']) + }) +}) + +describe('resolveActive / keep-alive tab heal', () => { + it('heals type-to-focus onto the visible main tab when a tile is buried', async () => { + // Repro for the reported main-tab miss: user typed in a session tile, then + // clicked the main/workspace tab without focusing its input. The tile stays + // mounted under data-pane-hidden, so activeTarget still reads tile:… and + // every type-to-focus request is dropped by the visible main composer. + mountSurface('tile:buried', true) + mountSurface('main') + markActiveComposer('tile:buried') + + expect(getActiveComposer()).toBe('main') + + const mainSaw: string[] = [] + const tileSaw: string[] = [] + + const off = onComposerFocusRequest(({ target }) => { + if (target === 'main') { + mainSaw.push(target) + } + + if (target === 'tile:buried') { + tileSaw.push(target) + } + }) + + requestComposerFocus('active', { typeChar: 'h' }) + await new Promise(resolve => window.setTimeout(resolve, 0)) + off() + + expect(mainSaw).toEqual(['main']) + expect(tileSaw).toEqual([]) + // Cache stays honest so dict/insert/Esc path all agree thereafter. + expect(getActiveComposer()).toBe('main') + }) + + it('keeps a live tile claim while that tile is the visible surface', () => { + mountSurface('main', true) + mountSurface('tile:front') + markActiveComposer('tile:front') + + expect(getActiveComposer()).toBe('tile:front') + }) + + it('heals an edit claim once the edit root is gone (no release site needed)', async () => { + mountSurface('main') + markActiveComposer('edit') + // No edit root in the document → claim is dead. getActiveComposer heals. + expect(getActiveComposer()).toBe('main') + + const mainSaw: string[] = [] + + const off = onComposerFocusRequest(({ target }) => { + if (target === 'main') { + mainSaw.push(target) + } + }) + + requestComposerFocus('active', { typeChar: 'a' }) + await new Promise(resolve => window.setTimeout(resolve, 0)) + off() + + expect(mainSaw).toEqual(['main']) + }) + + it('holds an edit claim while the edit composer root is mounted', () => { + const root = document.createElement('div') + root.dataset.slot = 'aui_edit-composer-root' + document.body.append(root) + mountSurface('main') + markActiveComposer('edit') + + expect(getActiveComposer()).toBe('edit') + }) +}) + +/** A chat surface inside a layout zone, mirroring ChatView-in-tree-group. */ +function mountZonedSurface(target: string, zone: string, hidden = false) { + const group = document.createElement('div') + group.dataset.treeGroup = zone + const layer = document.createElement('div') + layer.toggleAttribute('data-pane-hidden', hidden) + const surface = document.createElement('div') + surface.dataset.composerTarget = target + layer.append(surface) + group.append(layer) + document.body.append(group) + + return surface +} + +const collectModelMenuTargets = async (): Promise => { + const saw: string[] = [] + const off = onComposerModelMenuRequest(target => saw.push(target)) + + await new Promise(resolve => window.setTimeout(resolve, 0)) + off() + + return saw +} + +describe('requestModelMenuToggle', () => { + it('targets the pane under the pointer over the focused one (#74447 convention)', async () => { + mountZonedSurface('main', 'zone-a') + mountZonedSurface('tile:hovered', 'zone-b') + markActiveComposer('main') + $hoveredTreeGroup.set('zone-b') + + expect(requestModelMenuToggle()).toBe(true) + expect(await collectModelMenuTargets()).toEqual(['tile:hovered']) + }) + + it('falls back to the active composer when the pointer is off every zone', async () => { + mountZonedSurface('main', 'zone-a') + mountZonedSurface('tile:other', 'zone-b') + markActiveComposer('tile:other') + + expect(requestModelMenuToggle()).toBe(true) + expect(await collectModelMenuTargets()).toEqual(['tile:other']) + }) + + it('skips a hidden keep-alive tab in the hovered zone (targets its visible sibling)', async () => { + mountZonedSurface('main', 'zone-a', true) + mountZonedSurface('tile:front', 'zone-a') + markActiveComposer('main') + $hoveredTreeGroup.set('zone-a') + + expect(requestModelMenuToggle()).toBe(true) + expect(await collectModelMenuTargets()).toEqual(['tile:front']) + }) + + it('returns false with no chat surface on screen so the caller can open the dialog', async () => { + // Settings/profiles routes: no [data-composer-target] anywhere. + expect(requestModelMenuToggle()).toBe(false) + expect(await collectModelMenuTargets()).toEqual([]) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/focus.ts b/apps/desktop/src/app/chat/composer/focus.ts index a470edfcd958..2d403edef6ef 100644 --- a/apps/desktop/src/app/chat/composer/focus.ts +++ b/apps/desktop/src/app/chat/composer/focus.ts @@ -10,6 +10,9 @@ * steal focus from the composer effect. */ +import { queryAllVisible, queryVisible } from '@/components/pane-shell/pane-visibility' +import { $hoveredTreeGroup } from '@/components/pane-shell/tree/store' + import type { InlineRefInput } from './inline-refs' import { RICH_INPUT_SLOT } from './rich-editor' @@ -40,6 +43,22 @@ const INSERT_EVENT = 'hermes:composer-insert' const INSERT_REFS_EVENT = 'hermes:composer-insert-refs' const SUBMIT_EVENT = 'hermes:composer-submit' const VOICE_TOGGLE_EVENT = 'hermes:composer-voice-toggle' +const MODEL_MENU_EVENT = 'hermes:composer-model-menu' + +/** Inline edit composer root — mounted only while a user bubble is being edited. */ +const EDIT_COMPOSER_ROOT = '[data-slot="aui_edit-composer-root"]' + +/** Attribute-safe selector fragment. jsdom (vitest) does not ship `CSS.escape`. */ +const cssEscape = (value: string): string => { + if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') { + return CSS.escape(value) + } + + // Our targets are `'main'` / `'edit'` / `'tile:'` — alphanumerics plus `:` + // and `-`. Escape anything outside that set so a weird id cannot break the + // attribute selector. + return value.replace(/[^a-zA-Z0-9_:-]/g, ch => `\\${ch}`) +} interface SubmitDetail { target: ComposerTarget @@ -48,7 +67,76 @@ interface SubmitDetail { let activeTarget: ComposerTarget = 'main' -const resolve = (target: ComposerTarget | 'active') => (target === 'active' ? activeTarget : target) +/** + * The chat surface currently on screen (`data-composer-target` hung off each + * ChatView). Inactive tabs stay mounted with `data-pane-hidden`, so this uses + * the same visibility policy as every other document-wide surface lookup. + */ +const visibleChatTarget = (): ComposerTarget | null => { + if (typeof document === 'undefined') { + return null + } + + const surface = queryVisible('[data-composer-target]') + const target = surface?.dataset.composerTarget + + return target ? (target as ComposerTarget) : null +} + +/** True when `target` still has a live, on-screen subscriber. */ +const targetIsReachable = (target: ComposerTarget): boolean => { + if (typeof document === 'undefined') { + return true + } + + // The edit composer is an in-thread overlay, not a chat surface — it never + // stamps `data-composer-target`. While its root is mounted it still owns the + // bus; once it tears down the claim is dead. + if (target === 'edit') { + return Boolean(document.querySelector(EDIT_COMPOSER_ROOT)) + } + + // Exact match on a VISIBLE surface. Background keep-alive tabs carry the same + // `data-composer-target` but sit under `data-pane-hidden`, so queryVisible + // filters them out. + if (queryVisible(`[data-composer-target="${cssEscape(target)}"]`)) { + return true + } + + // A different chat surface is on screen → this claim is buried or gone. + // (A claim with zero stamped surfaces yet — first paint, pure-unit tests — + // keeps the marked key until the DOM contradicts it.) + if (queryVisible('[data-composer-target]')) { + return false + } + + return true +} + +/** + * The composer `'active'` should route to right now. + * + * The cached claim (`activeTarget`) wins while its surface is still on screen. + * Tab stacks keep inactive panes mounted, so focusing a tile then clicking the + * main tab leaves `activeTarget` pointing at a buried composer — with no + * subscriber on the visible surface, every type-to-focus keystroke is + * preventDefault'd and dropped. Heal to the visible chat surface (or main) + * whenever the claim is off-screen or gone, and keep the cache honest so Esc / + * voice / soft `/` agree with the keyboard path. + */ +const resolveActive = (): ComposerTarget => { + if (targetIsReachable(activeTarget)) { + return activeTarget + } + + const visible = visibleChatTarget() ?? 'main' + + activeTarget = visible + + return visible +} + +const resolve = (target: ComposerTarget | 'active') => (target === 'active' ? resolveActive() : target) const dispatch = (name: string, detail: T) => { if (typeof window === 'undefined') { @@ -80,9 +168,33 @@ export const markActiveComposer = (target: ComposerTarget) => { activeTarget = target } +/** Hand the routing key back when a composer unmounts, so `'active'` can never + * resolve to a composer that no longer has a subscriber — such a request is + * dispatched and then dropped by every mounted composer's target filter, and + * nothing re-marks the active composer on its own. + * + * Guarded on identity: a composer unmounting AFTER another one claimed the key + * (closing a background tile, a deferred edit-close cleanup) must not steal it + * from the live claimant. Falls through to {@link resolveActive} when the + * caller's surface is buried rather than gone, so closing on a tab switch that + * already re-fronted another chat surfaces there immediately. */ +export const releaseActiveComposer = (target: ComposerTarget) => { + if (activeTarget !== target) { + return + } + + // Prefer the visible chat surface over a hard `'main'` default — releasing a + // closed tile while another tile is fronted should land there, not the + // (possibly buried) workspace tab. + activeTarget = visibleChatTarget() ?? 'main' +} + /** The composer that last held focus — the target `'active'` resolves to. - * Used by broadcast listeners (voice, Esc-to-stop) to act on exactly one. */ -export const getActiveComposer = (): ComposerTarget => activeTarget + * Used by broadcast listeners (voice, Esc-to-stop) to act on exactly one. + * Heals a stale claim the same way {@link requestComposerFocus} does, so Esc + * and type-to-focus never disagree after a tab switch left the bus pointing at + * a keep-alive-mounted background composer. */ +export const getActiveComposer = (): ComposerTarget => resolveActive() export const requestComposerFocus = ( target: ComposerTarget | 'active' = 'active', @@ -148,6 +260,44 @@ export const requestVoiceToggle = (target: ComposerTarget | 'active' = 'active') export const onComposerVoiceToggleRequest = (handler: (target: ComposerTarget) => void) => subscribe<{ target: ComposerTarget }>(VOICE_TOGGLE_EVENT, ({ target }) => handler(target)) +/** The chat surface inside the zone the pointer is over, if any. Mirrors the + * tab verbs' hover-first targeting (`tabTargetGroupId`, #74447): the model + * hotkey lands in the pane you're pointing at without clicking into it first. + * Hidden keep-alive tabs are skipped like every document-wide lookup. */ +const composerTargetInHoveredZone = (): ComposerTarget | null => { + const zone = $hoveredTreeGroup.get() + + if (!zone || typeof document === 'undefined') { + return null + } + + const surface = queryAllVisible('[data-composer-target]').find( + el => el.closest('[data-tree-group]')?.dataset.treeGroup === zone + ) + + return (surface?.dataset.composerTarget as ComposerTarget | undefined) ?? null +} + +/** Toggle ONE composer's model menu — the `composer.modelPicker` hotkey. + * Targets the pane under the pointer first (the tab-verb convention), then + * the active composer. Returns false when no chat surface is on screen at + * all (settings, profiles…), so the caller can fall back to the full + * model-picker dialog instead of dispatching into the void. */ +export const requestModelMenuToggle = (): boolean => { + if (typeof document !== 'undefined' && !queryVisible('[data-composer-target]')) { + return false + } + + dispatch<{ target: ComposerTarget }>(MODEL_MENU_EVENT, { + target: composerTargetInHoveredZone() ?? resolveActive() + }) + + return true +} + +export const onComposerModelMenuRequest = (handler: (target: ComposerTarget) => void) => + subscribe<{ target: ComposerTarget }>(MODEL_MENU_EVENT, ({ target }) => handler(target)) + /** * Focus a composer input across React commit + browser focus restore. * @@ -175,9 +325,11 @@ export const focusComposerInput = (el: HTMLElement | null) => { window.setTimeout(focus, 0) } -/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.). */ +/** Drop focus from the main composer input (status-stack chrome, sidebar, etc.). + * Skips inactive tabs — they stay mounted, so an unscoped lookup can land on a + * background composer and leave the visible one focused. */ export const blurComposerInput = () => { - const el = document.querySelector(`[data-slot="${RICH_INPUT_SLOT}"]`) as HTMLElement | null + const el = queryVisible(`[data-slot="${RICH_INPUT_SLOT}"]`) if (el && document.activeElement === el) { el.blur() diff --git a/apps/desktop/src/app/chat/composer/hooks/use-at-completions.test.ts b/apps/desktop/src/app/chat/composer/hooks/use-at-completions.test.ts new file mode 100644 index 000000000000..cc1818f43b29 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-at-completions.test.ts @@ -0,0 +1,107 @@ +import { act, renderHook } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' + +import { queryClient } from '@/lib/query-client' + +import { useAtCompletions } from './use-at-completions' + +function gatewayStub(latencyMs = 40) { + const calls: string[] = [] + + const gateway = { + request: vi.fn(async (_method: string, params: { word: string }) => { + calls.push(params.word) + await new Promise(r => setTimeout(r, latencyMs)) + + return { items: [{ text: `@folder:${params.word.slice(1)}x/`, display: 'x/', meta: 'dir' }] } + }) + } + + return { calls, gateway } +} + +function setup(latencyMs = 40) { + const { calls, gateway } = gatewayStub(latencyMs) + + const { result } = renderHook(() => useAtCompletions({ gateway: gateway as never, sessionId: 's1', cwd: '/repo' })) + + return { calls, result } +} + +/** Type a burst of keystrokes `gapMs` apart, like a person. */ +async function type( + result: { current: { adapter: { search?: (q: string) => unknown } } }, + queries: string[], + gapMs: number +) { + for (const q of queries) { + act(() => { + result.current.adapter.search?.(q) + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(gapMs) + }) + } +} + +describe('PERF: @ path completions are cached and skip the debounce', () => { + it('serves a repeated query with no round trip and no spinner', async () => { + vi.useFakeTimers() + queryClient.clear() + + const { calls, result } = setup() + + // First visit to `apps/` pays the round trip. + await type(result, ['apps/'], 0) + await act(async () => { + await vi.advanceTimersByTimeAsync(200) + }) + + const afterFirst = calls.length + + expect(afterFirst).toBe(1) + + // Walk away and come back — Tab in, Backspace out, retype. Every one of + // these used to be a fresh git ls-files + rank on the backend. + await type(result, ['apps/desktop/', 'apps/', 'apps/desktop/', 'apps/'], 0) + await act(async () => { + await vi.advanceTimersByTimeAsync(200) + }) + + // Two distinct paths, so exactly two round trips total — the repeats are free. + expect(calls.length).toBe(2) + expect(result.current.loading).toBe(false) + + vi.useRealTimers() + }) + + it('a cached query paints without waiting out the debounce', async () => { + vi.useFakeTimers() + queryClient.clear() + + const { calls, result } = setup() + + await type(result, ['apps/'], 0) + await act(async () => { + await vi.advanceTimersByTimeAsync(200) + }) + + expect(calls.length).toBe(1) + + // Re-ask for the cached query and advance by far less than the 60ms + // debounce. A cached answer resolves in a microtask, so it must paint + // without the timer and without ever flipping the spinner on. + act(() => { + result.current.adapter.search?.('apps/') + }) + + await act(async () => { + await vi.advanceTimersByTimeAsync(1) + }) + + expect(result.current.loading).toBe(false) + expect(calls.length).toBe(1) + + vi.useRealTimers() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-at-completions.ts b/apps/desktop/src/app/chat/composer/hooks/use-at-completions.ts index d56a6d57e710..1f8a12df3365 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-at-completions.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-at-completions.ts @@ -1,7 +1,9 @@ import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-ui/core' import { useCallback } from 'react' +import { refChipLabel } from '@/components/assistant-ui/directive-text' import type { HermesGateway } from '@/hermes' +import { cachedPathCompletion, hasCachedPathCompletion } from '@/lib/slash-completion-cache' import { normalize } from '@/lib/text' import type { CompletionEntry, CompletionPayload } from './use-live-completion-adapter' @@ -60,7 +62,14 @@ function classify(entry: CompletionEntry): { return { type: kind, insertId: rest, - display: textValue(entry.display, rest || `@${kind}:`), + // The row shows exactly what picking it produces. Upstream keeps one + // label per item and hands it to the chip verbatim (DirectiveNode's + // `__label = item.label`); our wire format is `@kind:value`, which can't + // carry a label the way their `:type[label]{name=id}` does, so the same + // invariant is held by deriving both ends from refChipLabel. Without + // this the list said `desktop/`, the editor said `apps/desktop/`, and + // the chip said `desktop` — three names for one folder. + display: rest ? refChipLabel(kind, rest) : textValue(entry.display, `@${kind}:`), meta: textValue(entry.meta) } } @@ -82,6 +91,11 @@ export function useAtCompletions(options: { const { gateway, sessionId, cwd } = options const enabled = Boolean(gateway) + // Cache key: the completion depends on the query AND the directory it's + // resolved against, so a cwd or session change can't serve another tree's + // listing. + const cacheKey = useCallback((query: string) => `${cwd ?? ''}|${sessionId ?? ''}|${query}`, [cwd, sessionId]) + const fetcher = useCallback( async (query: string): Promise => { const starters = starterEntries(query) @@ -102,7 +116,15 @@ export function useAtCompletions(options: { } try { - const result = await gateway.request<{ items?: CompletionEntry[] }>('complete.path', params) + // De-duplicated the same way `/` completions are. Walking a path is + // inherently repetitive — Tab into a folder, Backspace out, retype a + // segment — and every one of those steps used to be a fresh + // `git ls-files` + rank on the backend (~40ms of the ~50ms round trip + // measured on this repo's 8k files). + const result = await cachedPathCompletion(cacheKey(query), () => + gateway.request<{ items?: CompletionEntry[] }>('complete.path', params) + ) + const items = result.items ?? [] return { items: items.length > 0 ? items : starters, query } @@ -110,7 +132,7 @@ export function useAtCompletions(options: { return { items: starters, query } } }, - [gateway, sessionId, cwd] + [cacheKey, gateway, sessionId, cwd] ) const toItem = useCallback((entry: CompletionEntry, index: number): Unstable_TriggerItem => { @@ -135,7 +157,13 @@ export function useAtCompletions(options: { } }, []) - return useLiveCompletionAdapter({ enabled, fetcher, toItem }) + // A query already in cache skips both the debounce and the loading state. + // This is what makes walking a tree feel instant rather than merely fast: + // the 60ms debounce exists to avoid a request per keystroke, and it buys + // nothing when the answer is already in hand. + const isCached = useCallback((query: string) => hasCachedPathCompletion(cacheKey(query)), [cacheKey]) + + return useLiveCompletionAdapter({ enabled, fetcher, isCached, toItem }) } /** Re-export `classify` for use by the formatter (insertion side). */ diff --git a/apps/desktop/src/app/chat/composer/hooks/use-auto-speak-replies.ts b/apps/desktop/src/app/chat/composer/hooks/use-auto-speak-replies.ts index 949b8f1020c2..aa657ca8291c 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-auto-speak-replies.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-auto-speak-replies.ts @@ -4,10 +4,11 @@ import { useEffect, useRef } from 'react' import { playSpeechText } from '@/lib/voice-playback' import { ownsAmbientCue } from '@/store/ambient' import { notifyError } from '@/store/notifications' -import { $messages } from '@/store/session' import { $voicePlayback } from '@/store/voice-playback' import { $autoSpeakReplies } from '@/store/voice-prefs' +import { useComposerScope } from '../scope' + interface AutoSpeakReply { id: string pending: boolean @@ -40,6 +41,9 @@ export function useAutoSpeakReplies({ sessionId }: UseAutoSpeakReplies) { const enabled = useStore($autoSpeakReplies) + // Wake on THIS composer's transcript: a tile subscribed to the primary's + // would never fire on its own replies (and would fire on someone else's). + const { $messages } = useComposerScope() const latest = useRef({ conversationActive, failureLabel, markSpoken, pendingReply }) latest.current = { conversationActive, failureLabel, markSpoken, pendingReply } @@ -83,5 +87,5 @@ export function useAutoSpeakReplies({ const stops = [$messages.subscribe(speakLatest), $voicePlayback.listen(speakLatest)] return () => stops.forEach(f => f()) - }, [enabled, sessionId]) + }, [$messages, enabled, sessionId]) } diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.test.tsx new file mode 100644 index 000000000000..7707c8ae7537 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.test.tsx @@ -0,0 +1,177 @@ +import { act, cleanup, render } from '@testing-library/react' +import { useLayoutEffect } from 'react' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { type ComposerAttachment, mainComposerScope, stashSessionDraft } from '@/store/composer' + +import type { QueueEditState } from '../composer-utils' +import { type ComposerTarget, getActiveComposer, markActiveComposer } from '../focus' +import { type ComposerScope, ComposerScopeProvider, MAIN_COMPOSER_SCOPE } from '../scope' + +import { useComposerDraft } from './use-composer-draft' + +const mockComposerApi = { setText: vi.fn() } + +vi.mock('@assistant-ui/react', () => ({ + useAui: () => ({ composer: () => mockComposerApi }), + useAuiState: (selector: (state: { composer: { text: string } }) => unknown) => selector({ composer: { text: '' } }), + useComposerRuntime: () => ({ + getState: () => ({ text: '' }), + subscribe: () => () => undefined + }) +})) + +interface ProbeHarnessProps { + activeQueueSessionKey: string | null + onLayoutSnapshot: (attachments: ComposerAttachment[]) => void + sessionId: string +} + +function ProbeHarness({ activeQueueSessionKey, onLayoutSnapshot, sessionId }: ProbeHarnessProps) { + useComposerDraft({ + activeQueueSessionKey, + focusKey: null, + inputDisabled: false, + queueEditRef: { current: null as QueueEditState | null }, + sessionId + }) + + // useLayoutEffect fires synchronously right after the DOM commit, BEFORE + // the hook's per-thread scope-swap useEffect (a passive effect) has a + // chance to swap attachmentScope.$attachments over to the new session. A + // synchronous read here — the same read ChatBar's `attachments` prop + // performs at render time — observes the OUTGOING session's attachments. + useLayoutEffect(() => { + onLayoutSnapshot(mainComposerScope.$attachments.get()) + }) + + return null +} + +describe('useComposerDraft — attachment scope stays coherent with the committed session on switch (#59305)', () => { + afterEach(() => { + cleanup() + mainComposerScope.clear() + }) + + it('clears the outgoing session attachments by the layout phase right after switching sessions', () => { + const attachmentA: ComposerAttachment = { id: 'url-A', kind: 'url', label: 'A' } + stashSessionDraft('session-A', 'hi from A', [attachmentA]) + + const snapshots: ComposerAttachment[][] = [] + + const { rerender } = render( + snapshots.push(s)} sessionId="session-A" /> + ) + + // Mount loads session A's stashed attachment into the (module-level) main + // scope — confirms the fixture actually seeded the leak precondition. + expect(mainComposerScope.$attachments.get()).toEqual([attachmentA]) + + snapshots.length = 0 // drop the initial-mount snapshot; only the switch matters + + act(() => { + rerender( + snapshots.push(s)} + sessionId="session-B" + /> + ) + }) + + // By the layout phase the scope must already be B's (empty) — a submit + // fired the instant B renders must never ship session A's attachment. + expect(snapshots[0]).toEqual([]) + }) +}) + +describe('useComposerDraft — rehydrate diagnostic log stays redacted', () => { + afterEach(() => { + cleanup() + mainComposerScope.clear() + vi.restoreAllMocks() + }) + + it('logs counts/kinds/scope on restore but never the raw url, refText, or label', () => { + const secretUrl = 'https://secret.example.com/private-workspace-path' + + const attachment: ComposerAttachment = { + id: 'url-secret', + kind: 'url', + label: 'do-not-leak-label', + refText: `@url:${secretUrl}` + } + + stashSessionDraft('session-secret', '', [attachment]) + + const debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => undefined) + + render( + undefined} + sessionId="session-secret" + /> + ) + + const rehydrateCalls = debugSpy.mock.calls.filter(call => call[0] === '[composer-rehydrate]') + expect(rehydrateCalls.length).toBeGreaterThan(0) + + const serialized = JSON.stringify(rehydrateCalls) + expect(serialized).not.toContain(secretUrl) + expect(serialized).not.toContain(attachment.label) + expect(serialized).not.toContain(attachment.refText) + + expect(rehydrateCalls[0]?.[1]).toMatchObject({ + attachmentCount: 1, + attachmentKinds: ['url'], + scope: 'session-secret' + }) + }) +}) + +describe('useComposerDraft — a closing composer hands the focus-bus key back', () => { + afterEach(() => { + cleanup() + mainComposerScope.clear() + markActiveComposer('main') + }) + + function renderScoped(target: ComposerTarget) { + const scope: ComposerScope = { ...MAIN_COMPOSER_SCOPE, target } + + return render( + + undefined} + sessionId="session-tile" + /> + + ) + } + + it('stops `active` resolving to a session tile once the tile unmounts', () => { + const { unmount } = renderScoped('tile:abc') + + // Mounting claims the bus for this tile — the leak precondition. + expect(getActiveComposer()).toBe('tile:abc') + + unmount() + + expect(getActiveComposer()).toBe('main') + }) + + it('leaves the key alone when another composer claimed it before this one unmounted', () => { + const { unmount } = renderScoped('tile:abc') + expect(getActiveComposer()).toBe('tile:abc') + + // The user clicks into a second tile, which claims the bus. + markActiveComposer('tile:other') + + unmount() + + expect(getActiveComposer()).toBe('tile:other') + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts index 11dc9534df0f..b5bf87dc4660 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-draft.ts @@ -1,7 +1,8 @@ import { useAui, useAuiState, useComposerRuntime } from '@assistant-ui/react' -import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' +import { type RefObject, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' import { SLASH_COMMAND_RE } from '@/lib/chat-runtime' +import { sanitizeComposerInput } from '@/lib/composer-input-sanitize' import { type ComposerAttachment, stashSessionDraft, takeSessionDraft } from '@/store/composer' import { isBrowsingHistory } from '@/store/composer-input-history' @@ -17,10 +18,17 @@ import { markActiveComposer, onComposerFocusRequest, onComposerInsertRefsRequest, - onComposerInsertRequest + onComposerInsertRequest, + releaseActiveComposer } from '../focus' import { type InlineRefInput, insertInlineRefsIntoEditor } from '../inline-refs' -import { composerPlainText, placeCaretEnd, renderComposerContents } from '../rich-editor' +import { + composerPlainText, + normalizeComposerEditorDom, + placeCaretEnd, + REF_RE, + renderComposerContents +} from '../rich-editor' import { useComposerScope } from '../scope' import type { ChatBarProps } from '../types' @@ -120,7 +128,7 @@ export function useComposerDraft({ const editor = editorRef.current if (editor) { - renderComposerContents(editor, next) + renderComposerContents(editor, next, { trailingCommitted: true }) placeCaretEnd(editor) } @@ -153,6 +161,15 @@ export function useComposerDraft({ } }, [focusInput, focusKey, focusRequestId, inputDisabled]) + // The mirror of the `markActiveComposer` above: give the key back when this + // composer goes away (a session tile closing, a pane unmounting). Covers both + // claim sites for this composer — `focusInput` here and ChatBar's `onFocus` — + // since they mark the same scope target. Without it `'active'` keeps + // resolving to a dead tile and every routed focus/insert request is dropped. + // (Heal-to-visible in focus.ts covers the keep-alive-tab case where the pane + // stays mounted behind the front tab; this covers true unmounts.) + useEffect(() => () => releaseActiveComposer(target), [target]) + useEffect(() => { if (inputDisabled) { return undefined @@ -189,6 +206,21 @@ export function useComposerDraft({ stashSessionDraft(scope, text, attachments) const loadIntoComposer = (text: string, attachments: ComposerAttachment[]) => { + // Diagnostic breadcrumb for #59305-class reports: identifies WHAT kind of + // state got restored into the composer (session switch, queue-edit + // restore, history browse) without logging any raw content. REF_RE has the + // global flag — testing against a throwaway clone avoids mutating the + // shared instance's lastIndex, which would otherwise corrupt this check on + // the next call. + if (attachments.length > 0 || new RegExp(REF_RE.source, REF_RE.flags).test(text)) { + console.debug('[composer-rehydrate]', { + attachmentCount: attachments.length, + attachmentKinds: attachments.map(a => a.kind), + hasTextRefs: new RegExp(REF_RE.source, REF_RE.flags).test(text), + scope: activeQueueSessionKeyRef.current + }) + } + attachmentScope.$attachments.set(cloneAttachments(attachments)) paintDraft(text, false) } @@ -212,7 +244,13 @@ export function useComposerDraft({ return draftRef.current } - const text = composerPlainText(editor) + // Same normalize-then-sanitize the rAF flush does. An emptied editor still + // holds the placeholder
that keeps the contenteditable from collapsing + // to a sliver, and that serializes as "\n" — so an editor the user just + // cleared would otherwise stash a one-newline draft and come back non-empty. + normalizeComposerEditorDom(editor) + + const text = sanitizeComposerInput(composerPlainText(editor)) if (text !== draftRef.current) { draftRef.current = text @@ -231,6 +269,7 @@ export function useComposerDraft({ // source otherwise), and (3) schedule the debounced per-session stash. // Browsing history / editing a queued prompt suppress the stash so recalled // text never clobbers the draft. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const sync = () => { const text = composerRuntime.getState().text @@ -239,7 +278,7 @@ export function useComposerDraft({ const editor = editorRef.current if (editor && document.activeElement !== editor && composerPlainText(editor) !== text) { - renderComposerContents(editor, text) + renderComposerContents(editor, text, { trailingCommitted: true }) } if (isBrowsingHistory(sessionIdRef.current) || queueEditRef.current) { @@ -318,7 +357,17 @@ export function useComposerDraft({ // Per-thread draft swap — the composer's only session coupling. Lifecycle // never clears composer state; this effect alone stashes on leave, restores // on enter. Keyed writes are idempotent, so no skip-sentinel. - useEffect(() => { + // + // MUST be a layout effect, not a passive one: it swaps attachmentScope's + // module-level $attachments atom, and a passive effect fires only after the + // browser paints the new session's view — leaving a window where the DOM + // already shows session B while $attachments (and therefore ChatBar's + // `attachments` prop) still holds session A's chips. A submit fired in that + // window (e.g. a fast session switch immediately followed by Enter) would + // ship A's attachments into B's turn (#59305). useLayoutEffect closes the + // window by running before paint. + + useLayoutEffect(() => { // A pending debounce timer from the outgoing session is now stale — its // scope was correct when scheduled, but the authoritative stash below // (and the cleanup on the way out) already covers that text. Letting it @@ -344,6 +393,7 @@ export function useComposerDraft({ // pagehide is load-bearing: React skips effect cleanups on reload, so Cmd+R // inside the debounce/rAF window would drop trailing keystrokes without this. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const flushPendingDraftPersist = () => { const scope = draftScopeRef.current @@ -381,6 +431,7 @@ export function useComposerDraft({ requestMainFocus, sessionIdRef, setComposerText, - stashAt + stashAt, + syncDraftFromEditor } } diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts index 318802bbab80..7353dc102e6b 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-metrics.ts @@ -1,14 +1,20 @@ import { useAuiState } from '@assistant-ui/react' import { type RefObject, useCallback, useEffect, useRef, useState } from 'react' +import { + chatSurfaceRoot, + clearSurfaceVar, + COMPOSER_HEIGHT_VAR, + COMPOSER_SURFACE_HEIGHT_VAR, + setSurfaceVar +} from '@/app/chat/surface-vars' import { useMediaQuery } from '@/hooks/use-media-query' import { useResizeObserver } from '@/hooks/use-resize-observer' -import { $composerPoppedOut } from '@/store/composer-popout' -import { isSecondaryWindow } from '@/store/windows' import { COMPOSER_COMPACT_PILL_PX, COMPOSER_SINGLE_LINE_MAX_PX, COMPOSER_STACK_BREAKPOINT_PX } from '../composer-utils' interface UseComposerMetricsArgs { + composerDockRef: RefObject composerRef: RefObject composerSurfaceRef: RefObject editorRef: RefObject @@ -23,7 +29,13 @@ interface UseComposerMetricsArgs { * tree's computed style, and `tight` only flips when it crosses the breakpoint. * Returns `stacked` (the only value the render needs). */ -export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, poppedOut }: UseComposerMetricsArgs): { +export function useComposerMetrics({ + composerDockRef, + composerRef, + composerSurfaceRef, + editorRef, + poppedOut +}: UseComposerMetricsArgs): { compactPill: boolean stacked: boolean } { @@ -76,31 +88,39 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, const lastBucketedSurfaceHeightRef = useRef(0) const lastTightRef = useRef(null) const lastCompactPillRef = useRef(null) + // Mirrored into a ref so `syncComposerMetrics` stays referentially stable — + // it's the shared ResizeObserver's handler, and a new identity every render + // would re-register the observation. + const poppedOutRef = useRef(poppedOut) + poppedOutRef.current = poppedOut const syncComposerMetrics = useCallback(() => { const composer = composerRef.current + // The dock is the full docked footprint — strips, status stack, composer — + // so it, not the composer alone, is what the thread has to clear. + const dock = composerDockRef.current - if (!composer) { + if (!composer || !dock) { return } // Floating composer is out of the thread's flow — it must not reserve any // bottom clearance. Zero the measured vars so the thread reclaims the space. - // (Read globals here so the callback stays stable; mirror the popoutAllowed - // gate since secondary windows are forced docked.) - if ($composerPoppedOut.get() && !isSecondaryWindow()) { - const root = document.documentElement + // Read through a ref so the callback stays stable, and read THIS surface's + // own state: pop-out is per layout zone, so a float in the left split must + // not zero the right split's clearance. + if (poppedOutRef.current) { lastBucketedHeightRef.current = 0 lastBucketedSurfaceHeightRef.current = 0 - root.style.setProperty('--composer-measured-height', '0px') - root.style.setProperty('--composer-surface-measured-height', '0px') + setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, '0px') + setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, '0px') return } - const { height, width } = composer.getBoundingClientRect() + const { height } = dock.getBoundingClientRect() + const { width } = composer.getBoundingClientRect() const surfaceHeight = composerSurfaceRef.current?.getBoundingClientRect().height - const root = document.documentElement if (width > 0) { const nextTight = width < COMPOSER_STACK_BREAKPOINT_PX @@ -135,7 +155,7 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, if (bucket !== lastBucketedHeightRef.current) { lastBucketedHeightRef.current = bucket - root.style.setProperty('--composer-measured-height', `${bucket}px`) + setSurfaceVar(composer, COMPOSER_HEIGHT_VAR, `${bucket}px`) } } @@ -144,12 +164,12 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, if (bucket !== lastBucketedSurfaceHeightRef.current) { lastBucketedSurfaceHeightRef.current = bucket - root.style.setProperty('--composer-surface-measured-height', `${bucket}px`) + setSurfaceVar(composer, COMPOSER_SURFACE_HEIGHT_VAR, `${bucket}px`) } } - }, [composerRef, composerSurfaceRef, editorRef]) + }, [composerDockRef, composerRef, composerSurfaceRef, editorRef]) - useResizeObserver(syncComposerMetrics, composerRef, composerSurfaceRef, editorRef) + useResizeObserver(syncComposerMetrics, composerDockRef, composerRef, composerSurfaceRef, editorRef) // Toggling pop-out changes whether the composer reserves thread clearance. // The ResizeObserver may not fire (the box can keep the same box size), so @@ -160,12 +180,16 @@ export function useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, }, [poppedOut, syncComposerMetrics]) useEffect(() => { + // Resolve the owning surface while the composer is still attached; the + // unmount cleanup runs after React detached the node, where closest() can + // no longer find [data-chat-surface]. + const root = chatSurfaceRoot(composerRef.current) + return () => { - const root = document.documentElement - root.style.removeProperty('--composer-measured-height') - root.style.removeProperty('--composer-surface-measured-height') + clearSurfaceVar(root, COMPOSER_HEIGHT_VAR) + clearSurfaceVar(root, COMPOSER_SURFACE_HEIGHT_VAR) } - }, []) + }, [composerRef]) // Pill compacts on real width (tile/pane), OR when stacked for any reason // (viewport-narrow / wrapped) so the controls row never over-runs. diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts index 0c2e4b61927c..8d1bf8c1ee56 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-placeholder.ts @@ -29,6 +29,7 @@ export function useComposerPlaceholder({ disabled, reconnecting, sessionId }: Us const prevSessionIdRef = useRef(sessionId) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const prev = prevSessionIdRef.current prevSessionIdRef.current = sessionId diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-popout.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-popout.ts index 518aa3658a56..b326025bb95d 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-popout.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-popout.ts @@ -1,18 +1,20 @@ import { useStore } from '@nanostores/react' -import { type RefObject, useCallback, useEffect } from 'react' +import { type RefObject, useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react' +import { usePaneGroup, usePaneVisible } from '@/components/pane-shell/pane-visibility' +import { useResizeObserver } from '@/hooks/use-resize-observer' import { triggerHaptic } from '@/lib/haptics' import { - $composerPopoutPosition, - $composerPoppedOut, + $composerPopoutZone, + clampPopoutPosition, + getComposerPopoutZone, + popoutBoundsElement, + type PopoutPosition, readPopoutBounds, - setComposerPopoutPosition, setComposerPoppedOut } from '@/store/composer-popout' import { isSecondaryWindow } from '@/store/windows' -import { useComposerScope } from '../scope' - import { useComposerPopoutGestures } from './use-popout-drag' interface UseComposerPopoutOptions { @@ -20,32 +22,122 @@ interface UseComposerPopoutOptions { } /** - * Pop-out engine: the docked↔floating state (a shared, persisted atom), the - * dock/float/toggle actions, the drag gestures, and the on-screen re-clamp. - * Secondary windows (the tiny Ctrl+Shift+N window, subagent watch windows) can't - * pop out — a floating composer makes no sense there and would yank the main - * window's composer out via the shared atom. + * This surface's on-screen placement, derived from its zone's drag intent. + * + * A zone stores one intent for its whole tab stack — drag the box in any tab and + * it moves in all of them — but each surface owns a different rect, so the + * intent is clamped per surface. Clamping into the store instead would have + * every keep-alive-mounted tab overwrite the others with a position bounded by + * ITS geometry, last writer winning: that's how a drag in one tab used to get + * lost in another. + * + * Re-placing is skipped while this surface drags (the gesture already clamped + * against this rect) and while it's an inactive tab (still mounted, so a live + * drag would otherwise force a reflow per background tab per frame). + */ +function usePopoutPlacement( + composerRef: RefObject, + groupId: string, + intent: PopoutPosition, + dragging: boolean, + poppedOut: boolean +): PopoutPosition { + const [placement, setPlacement] = useState(intent) + const visible = usePaneVisible() + // Re-place while this surface is the visible tab and isn't itself dragging. + const live = poppedOut && visible && !dragging + + // Resolved before the shared ResizeObserver below registers (hook order puts + // this layout effect first), so the observer always has this surface's own + // bounds element rather than a document-wide first match. + const boundsRef = useRef(null) + + useLayoutEffect(() => { + boundsRef.current = popoutBoundsElement(composerRef.current) + }) + + const reclamp = useCallback(() => { + const el = composerRef.current + + if (!el) { + return + } + + const size = { height: el.offsetHeight, width: el.offsetWidth } + const next = clampPopoutPosition(getComposerPopoutZone(groupId).position, size, readPopoutBounds(el)) + + // Bail on an unchanged placement: a sash drag resizes the surface every + // frame, and a fresh object each time re-renders the whole composer. + setPlacement(prev => (prev.bottom === next.bottom && prev.right === next.right ? prev : next)) + }, [composerRef, groupId]) + + // The surface resizing (sash drag, sidebar open, tab split) re-places the box + // against its new rect; the composer resizing (a growing draft) re-places it + // against its new height. + useResizeObserver( + useCallback(() => { + if (live) { + reclamp() + } + }, [live, reclamp]), + composerRef, + boundsRef + ) + + // useLayoutEffect, not useEffect: a tab revealed after the box was dragged in + // another one must not paint a frame at its stale placement before catching + // up. Runs before paint, and no-ops for hidden tabs (`live`). + useLayoutEffect(() => { + if (!live) { + return undefined + } + + reclamp() + // A second pass after layout settles (sidebar widths, fonts): anyone + // restored out of bounds is pulled back even if the first measure was + // premature. + const raf = requestAnimationFrame(reclamp) + window.addEventListener('resize', reclamp) + + return () => { + cancelAnimationFrame(raf) + window.removeEventListener('resize', reclamp) + } + }, [intent, live, reclamp]) + + return dragging ? intent : placement +} + +/** + * Pop-out engine: the docked↔floating state, the dock/float/toggle actions, the + * drag gestures, and this surface's placement. + * + * State is scoped to the surface's layout ZONE (its tab stack): tabs in the same + * zone share one float, so switching tabs keeps the box exactly where you put + * it, while a split zone beside them keeps its own — popping out on the left + * doesn't fling a composer out of the right. + * + * Secondary windows (the tiny Ctrl+Shift+N window, subagent watch windows) stay + * docked: a floating composer makes no sense in a scratch window. */ export function useComposerPopout({ composerRef }: UseComposerPopoutOptions) { - // The floating composer is a window-level singleton: only the main scope - // (not tiles) in a primary window may pop out. - const scope = useComposerScope() - const popoutAllowed = !isSecondaryWindow() && scope.popoutAllowed - const poppedOut = useStore($composerPoppedOut) && popoutAllowed - const popoutPosition = useStore($composerPopoutPosition) + const popoutAllowed = !isSecondaryWindow() + const groupId = usePaneGroup() + const zone = useStore(useMemo(() => $composerPopoutZone(groupId), [groupId])) + const poppedOut = zone.poppedOut && popoutAllowed const handleComposerPopOut = useCallback(() => { triggerHaptic('open') - setComposerPoppedOut(true) - }, []) + setComposerPoppedOut(groupId, true) + }, [groupId]) const handleComposerDock = useCallback(() => { triggerHaptic('success') - setComposerPoppedOut(false) - }, []) + setComposerPoppedOut(groupId, false) + }, [groupId]) // Double-click the grab area toggles dock/float. Undocking restores the last - // position (the persisted atom is never cleared on dock). + // position (a zone's stored position is never cleared on dock). const handleComposerToggle = useCallback(() => { poppedOut ? handleComposerDock() : handleComposerPopOut() }, [handleComposerDock, handleComposerPopOut, poppedOut]) @@ -56,39 +148,14 @@ export function useComposerPopout({ composerRef }: UseComposerPopoutOptions) { onPointerDown: onComposerGesturePointerDown } = useComposerPopoutGestures({ composerRef, + groupId, onDock: handleComposerDock, onPopOut: handleComposerPopOut, poppedOut, - position: popoutPosition + position: zone.position }) - // Keep the floating box on-screen: re-clamp (with the real measured size + - // thread bounds) when it pops out and on every window resize — so a position - // persisted on a bigger/other monitor, a shrunk window, or now-wider sidebar - // can never strand it. The rAF pass re-clamps after layout settles (sidebar - // widths, fonts), so anyone loading in out of bounds is pulled back + saved - // even if the first measure was premature. - useEffect(() => { - if (!poppedOut) { - return undefined - } - - const reclamp = (persist: boolean) => { - const el = composerRef.current - const size = el ? { height: el.offsetHeight, width: el.offsetWidth } : undefined - setComposerPopoutPosition($composerPopoutPosition.get(), { area: readPopoutBounds(el), persist, size }) - } - - reclamp(true) - const raf = requestAnimationFrame(() => reclamp(true)) - const onResize = () => reclamp(false) - window.addEventListener('resize', onResize) - - return () => { - cancelAnimationFrame(raf) - window.removeEventListener('resize', onResize) - } - }, [composerRef, poppedOut]) + const popoutPosition = usePopoutPlacement(composerRef, groupId, zone.position, dragging, poppedOut) return { dockProximity, diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts index 4e813f548aef..e1bb5442fef0 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-queue.ts @@ -106,7 +106,9 @@ export function useComposerQueue({ entryId: entry.id, sessionKey: activeQueueSessionKey }) - loadIntoComposer(entry.text, entry.attachments) + // Edit what the panel SHOWS. A queued `/skill` entry's text is the + // expanded skill body — never drop that into the composer. + loadIntoComposer(entry.displayText ?? entry.text, entry.attachments) triggerHaptic('selection') focusInput() } @@ -135,7 +137,7 @@ export function useComposerQueue({ if (next) { setQueueEditSnapshot({ ...queueEdit, entryId: next.id }) - loadIntoComposer(next.text, next.attachments) + loadIntoComposer(next.displayText ?? next.text, next.attachments) } else { setQueueEditSnapshot(null) loadIntoComposer(queueEdit.draft, queueEdit.attachments) @@ -213,6 +215,7 @@ export function useComposerQueue({ const accepted = await Promise.resolve( onSubmit(entry.text, { attachments: entry.attachments, + ...(entry.displayText ? { displayText: entry.displayText } : {}), fromQueue: true, sessionId: drainRuntimeSessionId, storedSessionId: drainQueueSessionKey @@ -322,6 +325,7 @@ export function useComposerQueue({ // never churns, so a change there is a real session switch and must NOT // migrate; only the runtime-derived key (queueSessionKey falsy → key is // sessionId) churns on a backend bounce/resume of the same conversation. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { const prev = prevQueueKeyRef.current prevQueueKeyRef.current = activeQueueSessionKey diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx index a09cd10ef29d..dd83d04f0c0f 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.test.tsx @@ -1,7 +1,9 @@ import { act, cleanup, renderHook, waitFor } from '@testing-library/react' import { afterEach, describe, expect, it, vi } from 'vitest' +import { $clarifyRequests } from '@/store/clarify' import type { ComposerAttachment } from '@/store/composer' +import { $gateway } from '@/store/gateway' import { useComposerSubmit } from './use-composer-submit' @@ -113,7 +115,9 @@ describe('useComposerSubmit busy-turn routing', () => { hook.result.current.submitDraft() }) - await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('/compress preserve context')) + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('/compress preserve context', { composerScope: 'stored-session' }) + ) expect(clearDraft).toHaveBeenCalledTimes(1) expect(onSteer).not.toHaveBeenCalled() expect(queueCurrentDraft).not.toHaveBeenCalled() @@ -159,9 +163,103 @@ describe('useComposerSubmit busy-turn routing', () => { hook.result.current.submitDraft() }) - await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('ordinary question', { attachments: [] })) + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('ordinary question', { + attachments: [], + composerScope: 'stored-session' + }) + ) expect(onSteer).not.toHaveBeenCalled() expect(queueCurrentDraft).not.toHaveBeenCalled() expect(onCancel).not.toHaveBeenCalled() }) + + it('threads the loaded composer scope through onSubmit for the #59305 submit-time guard', async () => { + const { hook, onSubmit } = renderSubmitHook({ text: 'hello' }) + + act(() => { + hook.result.current.submitDraft() + }) + + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('hello', expect.objectContaining({ composerScope: 'stored-session' })) + ) + }) +}) + +describe('useComposerSubmit with a clarify parked on the session', () => { + const gatewayRequest = vi.fn(async () => ({ ok: true })) + + const parkClarify = (sessionId: string) => { + $clarifyRequests.set({ + [sessionId]: { requestId: `req-${sessionId}`, question: 'which one?', choices: ['a', 'b'], sessionId } + }) + $gateway.set({ request: gatewayRequest } as unknown as ReturnType) + } + + afterEach(() => { + cleanup() + gatewayRequest.mockClear() + $clarifyRequests.set({}) + $gateway.set(null) + vi.restoreAllMocks() + }) + + it('skips the question and still sends the typed message on an idle session', async () => { + parkClarify('runtime-session') + const { hook, onSubmit } = renderSubmitHook({ text: 'actually do this instead' }) + + act(() => { + hook.result.current.submitDraft() + }) + + await waitFor(() => + expect(gatewayRequest).toHaveBeenCalledWith('clarify.respond', { + request_id: 'req-runtime-session', + answer: '' + }) + ) + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith('actually do this instead', expect.objectContaining({ attachments: [] })) + ) + expect($clarifyRequests.get()['runtime-session']).toBeUndefined() + }) + + it('skips the question before steering a busy turn', async () => { + parkClarify('runtime-session') + const { hook, onSteer } = renderSubmitHook({ busy: true, text: 'change course' }) + + act(() => { + hook.result.current.submitDraft() + }) + + await waitFor(() => expect(onSteer).toHaveBeenCalledWith('change course')) + expect(gatewayRequest).toHaveBeenCalledWith('clarify.respond', { request_id: 'req-runtime-session', answer: '' }) + }) + + it('leaves the question alone for an empty Enter (Stop, not an answer)', () => { + parkClarify('runtime-session') + const { hook, onCancel } = renderSubmitHook({ busy: true }) + + act(() => { + hook.result.current.submitDraft() + }) + + expect(gatewayRequest).not.toHaveBeenCalled() + expect($clarifyRequests.get()['runtime-session']).toBeDefined() + expect(onCancel).toHaveBeenCalledTimes(1) + }) + + it("leaves another session's question alone", async () => { + parkClarify('other-session') + const { hook, onSubmit } = renderSubmitHook({ text: 'unrelated message' }) + + act(() => { + hook.result.current.submitDraft() + }) + + await waitFor(() => expect(onSubmit).toHaveBeenCalled()) + expect(gatewayRequest).not.toHaveBeenCalled() + expect($clarifyRequests.get()['other-session']).toBeDefined() + }) }) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts index fb1cec2cf04a..9d34ddfca3f1 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-submit.ts @@ -2,12 +2,14 @@ import { type RefObject, useEffect, useRef } from 'react' import { SLASH_COMMAND_RE } from '@/lib/chat-runtime' import { triggerHaptic } from '@/lib/haptics' +import { hasClarifyRequest, skipClarifyRequest } from '@/store/clarify' import { clearSessionDraft, type ComposerAttachment } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' import { enqueueQueuedPrompt, type QueuedPromptEntry } from '@/store/composer-queue' import { cloneAttachments, type QueueEditState } from '../composer-utils' import { onComposerSubmitRequest } from '../focus' +import { pathifyRefs } from '../path-refs' import { composerPlainText } from '../rich-editor' import { useComposerScope } from '../scope' import type { ChatBarProps } from '../types' @@ -89,7 +91,11 @@ export function useComposerSubmit({ stashAt(submittedScope, text, submittedAttachments) } - void Promise.resolve(attachments ? onSubmit(text, { attachments }) : onSubmit(text)) + void Promise.resolve( + attachments + ? onSubmit(text, { attachments, composerScope: submittedScope }) + : onSubmit(text, { composerScope: submittedScope }) + ) .then(accepted => void (accepted === false ? restore() : clearSessionDraft(submittedScope))) .catch(restore) } @@ -134,9 +140,27 @@ export function useComposerSubmit({ } } - const text = draftRef.current + // A path that never got its committing space (`@apps/desktop/` left by a Tab + // descend, then Enter) is still the reference the user picked — promote it + // on the way out so it attaches instead of submitting as inert text. + const text = pathifyRefs(draftRef.current) const payloadPresent = text.trim().length > 0 || attachments.length > 0 + // A clarify card parked on this session owns the turn: the agent is blocked + // inside its tool batch waiting on `clarify.respond`, so a follow-up routed + // through steer/queue sits undelivered until the clarify's own timeout + // (default 5 min) — the message looks sent and nothing happens. Typing a + // real message instead of picking an option IS the answer "none of these": + // skip the question so the tool returns, then route the words normally. + // + // Fire-and-forget, not awaited: the skip clears the card synchronously and + // both RPCs ride the same socket in call order, so the gateway resolves the + // clarify before it sees the follow-up. Awaiting first would leave the draft + // live for a tick — long enough for a second Enter to send it twice. + if (payloadPresent && !queueEdit && hasClarifyRequest(sessionId)) { + void skipClarifyRequest(sessionId) + } + if (queueEdit) { exitQueuedEdit('save') } else if (busy) { diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.test.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.test.ts new file mode 100644 index 000000000000..44c010cb4f11 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.test.ts @@ -0,0 +1,294 @@ +import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-ui/core' +import { act, renderHook } from '@testing-library/react' +import { createRef } from 'react' +import { describe, expect, it, vi } from 'vitest' + +import { composerPlainText, renderComposerContents, RICH_INPUT_SLOT } from '../rich-editor' + +import { useComposerTrigger } from './use-composer-trigger' + +/** A live contentEditable seeded with `text`, caret parked at the end. */ +function mountEditor(text: string) { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.contentEditable = 'true' + document.body.append(editor) + renderComposerContents(editor, text) + + const range = document.createRange() + range.selectNodeContents(editor) + range.collapse(false) + const selection = window.getSelection()! + selection.removeAllRanges() + selection.addRange(range) + + return editor +} + +const item = (command: string, group = 'Skills'): Unstable_TriggerItem => ({ + id: command, + type: 'slash', + label: command.slice(1), + metadata: { command, display: command, meta: '', group, action: '', rawText: command } +}) + +function mountTrigger(editor: HTMLDivElement, items: Unstable_TriggerItem[]) { + const editorRef = createRef() as { current: HTMLDivElement | null } + editorRef.current = editor + + const draftRef = { current: composerPlainText(editor) } + + const adapter: Unstable_TriggerAdapter = { + categories: () => [], + categoryItems: () => [], + search: () => items + } + + const setComposerText = vi.fn() + + const hook = renderHook(() => + useComposerTrigger({ + at: { adapter: null, loading: false }, + draftRef, + editorRef, + requestMainFocus: vi.fn(), + setComposerText, + slash: { adapter, loading: false } + }) + ) + + return { draftRef, hook, setComposerText } +} + +describe('useComposerTrigger — slash anywhere in the prompt', () => { + it('opens the completion list for a slash typed mid-message', () => { + const editor = mountEditor('please run /cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toMatchObject({ kind: '/', inline: true, query: 'cle' }) + expect(hook.result.current.triggerItems).toHaveLength(1) + }) + + it('inserts the picked skill inline and keeps the surrounding prose intact', () => { + const editor = mountEditor('please run /cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.replaceTriggerWithChip(item('/clean'))) + + // The `/cle` the user typed is replaced by the full command; "please run" + // in front of it survives untouched. + expect(composerPlainText(editor)).toBe('please run /clean ') + }) + + it('offers only skills mid-message, not app commands', () => { + // `/model` and `/new` act on the app — meaningless as a reference in prose. + const editor = mountEditor('please run /') + const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands'), item('/new', 'Commands')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean']) + }) + + it('still offers the full command set at the start of the prompt', () => { + const editor = mountEditor('/') + const { hook } = mountTrigger(editor, [item('/clean'), item('/model', 'Commands')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.triggerItems.map(i => i.label)).toEqual(['clean', 'model']) + }) + + it('still opens the list for a slash at the start of the prompt', () => { + const editor = mountEditor('/cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toMatchObject({ kind: '/', query: 'cle' }) + expect(hook.result.current.trigger?.inline).toBeUndefined() + }) + + it('leaves a mid-message file path alone', () => { + const editor = mountEditor('open src/foo/bar') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toBeNull() + }) + + it('opens the list for a second slash after a leading command', () => { + // `/work /cle`: the command regex's argument tail would otherwise swallow + // `/cle` as an argument to `/work`, and a no-arg command suppresses the + // popover — so every slash after the first went dead. + const editor = mountEditor('/work /cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.trigger).toMatchObject({ kind: '/', inline: true, query: 'cle' }) + expect(hook.result.current.triggerItems).toHaveLength(1) + }) + + it('inserts the second command without disturbing the first', () => { + const editor = mountEditor('/work rewrite the composer /cle') + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.replaceTriggerWithChip(item('/clean'))) + + expect(composerPlainText(editor)).toBe('/work rewrite the composer /clean ') + }) +}) + +describe('useComposerTrigger — free-text slash arguments', () => { + it('keeps a picked /goal command as editable text while retaining subcommand completion', () => { + const editor = mountEditor('/go') + const goal = item('/goal', 'Commands') + const { hook } = mountTrigger(editor, [goal]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.replaceTriggerWithChip(goal)) + + expect(composerPlainText(editor)).toBe('/goal ') + expect(editor.querySelector('[data-slash-kind]')).toBeNull() + expect(hook.result.current.trigger).not.toBeNull() + }) + + it('does not seal a multi-word /goal into a chip when the option list runs empty', () => { + const editor = mountEditor('/goal finish the full prompt') + const { hook } = mountTrigger(editor, []) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.slashFreeTextArgStage).toBe(true) + expect(hook.result.current.commitTypedSlashDirective()).toBe(false) + expect(composerPlainText(editor)).toBe('/goal finish the full prompt') + expect(editor.querySelector('[data-slash-kind]')).toBeNull() + }) + + it('treats the default highlight as a suggestion until the user arrows to a row', () => { + const editor = mountEditor('/goal stat') + const { hook } = mountTrigger(editor, [item('/goal status', 'Options')]) + + act(() => hook.result.current.refreshTrigger()) + expect(hook.result.current.triggerActiveExplicit).toBe(false) + + act(() => hook.result.current.moveTriggerActive(1)) + expect(hook.result.current.triggerActiveExplicit).toBe(true) + }) + + it('drops a deliberate selection once the query moves on', () => { + const editor = mountEditor('/goal stat') + const { hook } = mountTrigger(editor, [item('/goal status', 'Options')]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.moveTriggerActive(1)) + + renderComposerContents(editor, '/goal start the migration') + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.triggerActiveExplicit).toBe(false) + }) + + it('keeps a multi-word /resume search typeable instead of firing the picker action', () => { + // The session list always ends in a "Browse all sessions…" action row, so + // an accept here doesn't insert a chip — it empties the composer and opens + // the overlay, taking the half-typed query with it. + const editor = mountEditor('/resume my new') + const { hook } = mountTrigger(editor, [item('/resume', 'Sessions')]) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.slashFreeTextArgStage).toBe(true) + expect(hook.result.current.commitTypedSlashDirective()).toBe(false) + expect(composerPlainText(editor)).toBe('/resume my new') + }) + + it('still commits a fully typed finite option as one directive chip', () => { + const editor = mountEditor('/personality creative') + const { hook } = mountTrigger(editor, []) + + act(() => hook.result.current.refreshTrigger()) + + expect(hook.result.current.slashFreeTextArgStage).toBe(false) + act(() => { + expect(hook.result.current.commitTypedSlashDirective()).toBe(true) + }) + expect(composerPlainText(editor)).toBe('/personality creative ') + expect(editor.querySelector('[data-slash-kind]')?.getAttribute('data-ref-text')).toBe('/personality creative') + }) +}) + +describe('useComposerTrigger — chip survival (the plaintext-demotion bug class)', () => { + it('keeps a leading command pill through a Backspace path-ascend', () => { + // The reported repro: `/work @folder…` then Backspace — both chips went + // plaintext because ascend re-rendered the whole editor from text. + const editor = mountEditor('/work @Desktop/') + const { hook } = mountTrigger(editor, []) + + expect(editor.querySelector('[data-slash-kind]')).not.toBeNull() + + act(() => hook.result.current.refreshTrigger()) + expect(hook.result.current.trigger).toMatchObject({ kind: '@', query: 'Desktop/' }) + + let ran = false + act(() => { + ran = hook.result.current.ascendTriggerPath() + }) + + expect(ran).toBe(true) + expect(composerPlainText(editor)).toBe('/work @') + expect(editor.querySelector('[data-slash-kind]')).not.toBeNull() + }) + + it('keeps a leading command pill when a folder pick commits its ref chip', () => { + const editor = mountEditor('/work @Desk') + + const folder: Unstable_TriggerItem = { + id: 'folder:Desktop', + type: 'folder', + label: 'Desktop', + metadata: { rawText: '@folder:Desktop', insertId: 'Desktop' } + } + + const { hook } = mountTrigger(editor, [folder]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.replaceTriggerWithChip(folder)) + + expect(composerPlainText(editor)).toBe('/work @folder:`Desktop` ') + expect(editor.querySelector('[data-slash-kind]')).not.toBeNull() + expect(editor.querySelector('[data-ref-kind="folder"]')).not.toBeNull() + }) + + it('commits in place when Chromium has split the token across text nodes', () => { + // Chromium fragments text nodes around contenteditable=false chips; the + // commit path must span the fragments instead of bailing to a full + // re-render. + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.contentEditable = 'true' + document.body.append(editor) + editor.append(document.createTextNode('please run /c'), document.createTextNode('le')) + + const caret = document.createRange() + caret.setStart(editor.lastChild!, 2) + caret.collapse(true) + const selection = window.getSelection()! + selection.removeAllRanges() + selection.addRange(caret) + + const { hook } = mountTrigger(editor, [item('/clean')]) + + act(() => hook.result.current.refreshTrigger()) + act(() => hook.result.current.replaceTriggerWithChip(item('/clean'))) + + expect(composerPlainText(editor)).toBe('please run /clean ') + expect(editor.querySelector('[data-slash-kind]')).not.toBeNull() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts index 20abc03309f5..fae1d03a3133 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-trigger.ts @@ -2,18 +2,69 @@ import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-u import { type MutableRefObject, type RefObject, useCallback, useEffect, useRef, useState } from 'react' import { hermesDirectiveFormatter } from '@/components/assistant-ui/directive-text' -import { desktopSlashCommandTakesArgs } from '@/lib/desktop-slash-commands' +import { desktopSlashCommandArgumentMode } from '@/lib/desktop-slash-commands' -import { COMPLETION_ACTIONS, slashArgStage, slashChipKindForItem, slashCommandToken } from '../composer-utils' import { + COMPLETION_ACTIONS, + isSkillItem, + slashArgStage, + slashChipKindForItem, + slashCommandToken +} from '../composer-utils' +import { + appendComposerContents, + caretOffsetInEditor, composerPlainText, - placeCaretEnd, + placeCaretAtOffset, refChipElement, renderComposerContents, + replaceBeforeCaret, + RICH_INPUT_SLOT, slashChipElement } from '../rich-editor' import { detectTrigger, textBeforeCaret, type TriggerState } from '../text-utils' +/** + * Rebuild-from-text fallback for carets the range walk can't anchor (a + * non-collapsed selection, a caret not preceded by contiguous text). It + * re-renders the whole editor from serialized text, so it only runs when the + * in-place path reports failure — never as the default. + * + * The split is around the CARET, not the end of the draft. Slicing + * `length - tokenLength` off the end assumed the trigger token was the last + * thing in the editor: a completion picked mid-message chopped the trailing + * prose off and stranded a partial `folder:` in front of the chip, because the + * window it removed wasn't the token the user was typing. + */ +export function rebuildAroundCaret(editor: HTMLDivElement, tokenLength: number, insert: DocumentFragment | string) { + const current = composerPlainText(editor) + const caret = caretOffsetInEditor(editor) + const prefix = current.slice(0, Math.max(0, caret - tokenLength)) + const suffix = current.slice(caret) + + if (typeof insert === 'string') { + renderComposerContents(editor, `${prefix}${insert}${suffix}`) + placeCaretAtOffset(editor, prefix.length + insert.length) + + return + } + + // Measure before appending — moving a fragment empties it. Appending the + // element rather than re-serializing keeps mid-message slash pills alive: + // they have no text hydration, unlike `@` refs and the leading command. + const scratch = document.createElement('div') + + scratch.dataset.slot = RICH_INPUT_SLOT + scratch.append(insert.cloneNode(true)) + + const inserted = composerPlainText(scratch) + + renderComposerContents(editor, prefix) + editor.append(insert) + appendComposerContents(editor, suffix) + placeCaretAtOffset(editor, prefix.length + inserted.length) +} + interface CompletionSource { adapter: Unstable_TriggerAdapter | null loading: boolean @@ -23,6 +74,10 @@ interface UseComposerTriggerOptions { at: CompletionSource draftRef: MutableRefObject editorRef: RefObject + /** `:joy` emoji completions — inserts the emoji character, never a chip. */ + emoji?: CompletionSource + /** Bank the pre-commit state so a popover pick is a single undo step. */ + recordUndoPoint?: () => void requestMainFocus: () => void setComposerText: (text: string) => void slash: CompletionSource @@ -41,12 +96,19 @@ export function useComposerTrigger({ at, draftRef, editorRef, + emoji, + recordUndoPoint, requestMainFocus, setComposerText, slash }: UseComposerTriggerOptions) { const [trigger, setTrigger] = useState(null) const [triggerActive, setTriggerActive] = useState(0) + // The list highlights its first row on open, which is a suggestion rather + // than a choice. This records that the user moved the highlight themselves, + // which is what lets Enter accept a completion in a free-text argument stage + // without stealing prose from everyone who never touched the arrows. + const [triggerActiveExplicit, setTriggerActiveExplicit] = useState(false) const [triggerItems, setTriggerItems] = useState([]) // Set synchronously in keydown when the open trigger popover consumes a // navigation/control key (Arrow/Enter/Tab/Escape). The subsequent keyup must @@ -57,6 +119,11 @@ export function useComposerTrigger({ // re-rendered and the handler closure sees the post-keydown state. const triggerKeyConsumedRef = useRef(false) + const resetTriggerActive = useCallback(() => { + setTriggerActive(0) + setTriggerActiveExplicit(false) + }, []) + const refreshTrigger = useCallback(() => { const editor = editorRef.current @@ -71,10 +138,10 @@ export function useComposerTrigger({ // is present do we pay the cost of the full walk + DOM range work. const rawText = editor.textContent ?? '' - if (!rawText.includes('@') && !rawText.includes('/')) { + if (!rawText.includes('@') && !rawText.includes('/') && !rawText.includes(':')) { if (trigger) { setTrigger(null) - setTriggerActive(0) + resetTriggerActive() } return @@ -83,11 +150,16 @@ export function useComposerTrigger({ const before = textBeforeCaret(editor) const found = detectTrigger(before ?? composerPlainText(editor)) - // The arg-stage popover is only useful for commands with an options screen. - // For a no-arg command it would dead-end on "No matches", so drop it — the - // directive is already complete. + // A text-only command has no completion screen once its prose begins. Mixed + // commands such as /goal stay live so their finite subcommands can still be + // suggested, while arbitrary goal text remains valid. + const argumentMode = + found?.kind === '/' && slashArgStage(found.query) + ? desktopSlashCommandArgumentMode(slashCommandToken(found.query)) + : null + const detected = - found?.kind === '/' && slashArgStage(found.query) && !desktopSlashCommandTakesArgs(slashCommandToken(found.query)) + found?.kind === '/' && slashArgStage(found.query) && argumentMode !== 'options' && argumentMode !== 'mixed' ? null : found @@ -98,12 +170,18 @@ export function useComposerTrigger({ // caret move (mouseup) or a stray refresh — must preserve the user's // current selection instead of snapping back to the first item. if (detected?.kind !== trigger?.kind || detected?.query !== trigger?.query) { - setTriggerActive(0) + resetTriggerActive() } - }, [editorRef, trigger]) + }, [editorRef, resetTriggerActive, trigger]) const triggerAdapter: Unstable_TriggerAdapter | null = - trigger?.kind === '@' ? at.adapter : trigger?.kind === '/' ? slash.adapter : null + trigger?.kind === '@' + ? at.adapter + : trigger?.kind === '/' + ? slash.adapter + : trigger?.kind === ':' + ? (emoji?.adapter ?? null) + : null useEffect(() => { if (!trigger || !triggerAdapter?.search) { @@ -112,20 +190,46 @@ export function useComposerTrigger({ return } - setTriggerItems(triggerAdapter.search(trigger.query)) + const items = triggerAdapter.search(trigger.query) + + // Mid-message only offers SKILLS. A built-in like `/model` or `/new` acts + // on the app, so it's meaningless as a reference inside prose — only a + // skill reads as "handle this part with X". Filtering here rather than in + // the fetcher keeps one completion source for both shapes. + setTriggerItems(trigger.inline ? items.filter(isSkillItem) : items) }, [trigger, triggerAdapter]) - const triggerLoading = trigger?.kind === '@' ? at.loading : trigger?.kind === '/' ? slash.loading : false + const triggerLoading = + trigger?.kind === '@' + ? at.loading + : trigger?.kind === '/' + ? slash.loading + : trigger?.kind === ':' + ? (emoji?.loading ?? false) + : false // Suppress the "No matches" empty state once a slash command is past its name: // a no-arg command has nothing to offer, and a fully-typed arg commits on // Space/Tab — neither should dead-end on a popover. const argStageEmpty = trigger?.kind === '/' && slashArgStage(trigger.query) && !triggerLoading && !triggerItems.length + const slashArgumentMode = + trigger?.kind === '/' && slashArgStage(trigger.query) + ? desktopSlashCommandArgumentMode(slashCommandToken(trigger.query)) + : null + + const slashFreeTextArgStage = slashArgumentMode === 'mixed' || slashArgumentMode === 'text' + const closeTrigger = () => { setTrigger(null) setTriggerItems([]) - setTriggerActive(0) + resetTriggerActive() + } + + /** Step the highlight, marking it as the user's own deliberate pick. */ + const moveTriggerActive = (delta: number) => { + setTriggerActiveExplicit(true) + setTriggerActive(idx => (idx + delta + triggerItems.length) % triggerItems.length) } useEffect(() => { @@ -136,9 +240,16 @@ export function useComposerTrigger({ // the completion list is empty because the arg is already fully typed (the // backend completer drops exact matches). Reuses the chip path via a // synthetic item whose serialized form is the verbatim text. - const commitTypedSlashDirective = () => { + const commitTypedSlashDirective = (): boolean => { if (trigger?.kind !== '/') { - return + return false + } + + // Free prose must stay ordinary contentEditable text. This guard also + // protects against a stale completion result reaching the keydown path + // before refreshTrigger has caught up with the latest DOM input. + if (desktopSlashCommandArgumentMode(slashCommandToken(trigger.query)) !== 'options') { + return false } const text = `/${trigger.query.trimEnd()}` @@ -156,26 +267,33 @@ export function useComposerTrigger({ rawText: text } }) + + return true } - const replaceTriggerWithChip = (item: Unstable_TriggerItem) => { + const replaceTriggerWithChip = (item: Unstable_TriggerItem, options?: { descend?: boolean }) => { const editor = editorRef.current if (!editor || !trigger) { return } + // Bank the pre-commit state first — every path below mutates the editor, + // and a pick must be exactly one undo step. + recordUndoPoint?.() + + const rebuildAround = (insert: DocumentFragment | string) => rebuildAroundCaret(editor, trigger.tokenLength, insert) + // Action items (e.g. "Browse all sessions…") run a side effect instead of // inserting a chip: strip the typed trigger token, then fire the action. const completionAction = (item.metadata as { action?: unknown } | undefined)?.action const runAction = typeof completionAction === 'string' ? COMPLETION_ACTIONS[completionAction] : undefined if (runAction) { - const current = composerPlainText(editor) - const prefix = current.slice(0, Math.max(0, current.length - trigger.tokenLength)) + if (!replaceBeforeCaret(editor, trigger.tokenLength, document.createDocumentFragment())) { + rebuildAround('') + } - renderComposerContents(editor, prefix) - placeCaretEnd(editor) draftRef.current = composerPlainText(editor) setComposerText(draftRef.current) closeTrigger() @@ -188,93 +306,151 @@ export function useComposerTrigger({ const serialized = hermesDirectiveFormatter.serialize(item) const starter = serialized.endsWith(':') + // Tab on a folder walks INTO it instead of committing it: re-type the + // token as the bare path so the next `complete.path` lists that folder's + // children, exactly as typing the path by hand would. Enter still commits + // the folder itself — the two intents are distinct, so the keys are too. + // Only `@` folders descend; a slash command's arg list has no hierarchy. + const descendInto = + options?.descend && trigger.kind === '@' && item.type === 'folder' + ? String((item.metadata as { insertId?: unknown } | undefined)?.insertId ?? '') + : '' + + const finish = (keepOpen: boolean) => { + draftRef.current = composerPlainText(editor) + setComposerText(draftRef.current) + requestMainFocus() + keepOpen ? window.setTimeout(refreshTrigger, 0) : closeTrigger() + } + + if (descendInto) { + const path = descendInto.endsWith('/') ? descendInto : `${descendInto}/` + // Carry the browse scope down with the path. Dropping it turned an + // explicit `@folder:` browse into a bare `@apps/desktop/` token halfway + // through, so the next completion silently widened back to files and the + // committed chip had to re-guess the kind from a trailing slash. + const scope = trigger.scope ? `${trigger.scope}:` : '' + const fragment = document.createDocumentFragment() + + fragment.append(document.createTextNode(`@${scope}${path}`)) + + if (!replaceBeforeCaret(editor, trigger.tokenLength, fragment)) { + rebuildAround(`@${scope}${path}`) + } + + return finish(true) + } + // Picking a bare arg-taking command (e.g. `/personality`) shouldn't commit // it — expand to its options step so the popover shows the inline list, just // as typing `/personality ` by hand would. A serialized value with a space is - // already an arg pick (`/personality alice`), so it commits normally. + // already an arg pick (`/personality alice`), so it commits normally. An + // inline (mid-message) pick never expands: it's a reference inside prose, so + // there's no command invocation for the args to belong to. const command = (item.metadata as { command?: string } | undefined)?.command ?? '' - const expandsToArgs = trigger.kind === '/' && !serialized.includes(' ') && desktopSlashCommandTakesArgs(command) + const argumentMode = desktopSlashCommandArgumentMode(command) + const expandsToArgs = trigger.kind === '/' && !trigger.inline && !serialized.includes(' ') && argumentMode !== null const text = starter || serialized.endsWith(' ') ? serialized : `${serialized} ` const directive = !starter && serialized.match(/^@([^:]+):(.+)$/) // No pill while expanding — the bare command stays plain text until an arg // is picked, at which point a single pill is emitted for the full command. const slashKind = !expandsToArgs && trigger.kind === '/' ? slashChipKindForItem(item) : null - const keepTriggerOpen = starter || expandsToArgs + const keepTriggerOpen = starter || (expandsToArgs && argumentMode !== 'text') - const finish = () => { - draftRef.current = composerPlainText(editor) - setComposerText(draftRef.current) - requestMainFocus() - keepTriggerOpen ? window.setTimeout(refreshTrigger, 0) : closeTrigger() - } + const chip = slashKind + ? slashChipElement(serialized, slashKind) + : directive + ? // Carry the picked row's own label into the chip rather than letting + // it re-derive one from the value. Upstream's DirectiveNode does the + // same (`__label = item.label`), and it's what makes the list and the + // chip agree: you get the string you just read, not a second guess at + // it. Falls back to the shared deriver for callers with no label. + refChipElement(directive[1], directive[2], (item.metadata as { display?: string })?.display || item.label) + : null - const sel = window.getSelection() - const range = sel?.rangeCount ? sel.getRangeAt(0) : null - const node = range?.startContainer - const offset = range?.startOffset ?? 0 + // The trailing space is a convenience for "keep typing after the chip", so + // it's wrong when the caret already has whitespace in front of it — a pick + // made mid-sentence would leave a double space in the prose. + const followedBySpace = /^\s/.test(composerPlainText(editor).slice(caretOffsetInEditor(editor))) + const fragment = document.createDocumentFragment() + + chip + ? fragment.append(chip, ...(followedBySpace ? [] : [document.createTextNode(' ')])) + : fragment.append(document.createTextNode(followedBySpace ? text.trimEnd() : text)) + + if (!replaceBeforeCaret(editor, trigger.tokenLength, fragment)) { + // The failed in-place attempt never consumed the fragment, so the chip + + // trailing space are re-inserted around the caret here. Moving the + // element (rather than re-serializing) keeps mid-message slash pills + // alive — they have no text hydration, unlike `@` refs and the leading + // command. + rebuildAround(chip ? fragment : text) + } - if (!sel || !range || node?.nodeType !== Node.TEXT_NODE || offset < trigger.tokenLength) { - const current = composerPlainText(editor) - const prefix = current.slice(0, Math.max(0, current.length - trigger.tokenLength)) + finish(keepTriggerOpen) + } - if (slashKind) { - // Two-step arg picks (e.g. `/handoff` pill already inserted, now picking - // the platform) land here because the caret sits past a contenteditable - // chip. Rebuild the prefix and re-emit a single pill for the full command. - renderComposerContents(editor, prefix) - editor.append(slashChipElement(serialized, slashKind), document.createTextNode(' ')) - placeCaretEnd(editor) + /** Backspace inside an `@` path drops the last segment (`a/b/` → `a/`) + * instead of one character, and once the path is empty it drops the browse + * scope (`@folder:` → `@`) rather than nibbling `:`, `r`, `e`, `d`… back + * through the directive syntax the user never typed. Descending is one Tab + * per level, so climbing back out costs one key per level too. Returns + * false when the caret isn't in a path, so keydown falls through. */ + const ascendTriggerPath = () => { + const editor = editorRef.current - return finish() - } + if (!editor || trigger?.kind !== '@') { + return false + } - renderComposerContents(editor, `${prefix}${text}`) - placeCaretEnd(editor) + const scope = trigger.scope ? `${trigger.scope}:` : '' - return finish() + if (!trigger.value.includes('/') && !scope) { + return false } - const replaceRange = document.createRange() - replaceRange.setStart(node, offset - trigger.tokenLength) - replaceRange.setEnd(node, offset) - replaceRange.deleteContents() + // Trailing slash means we're listing a folder's children: drop that + // folder. Otherwise a partial segment is typed — drop just that. With the + // value already empty, the only thing left to drop is the scope itself. + const trimmed = trigger.value.replace(/\/$/, '') + const parent = trimmed.slice(0, trimmed.lastIndexOf('/') + 1) + const next = trigger.value ? `${scope}${parent}` : '' - const chip = slashKind - ? slashChipElement(serialized, slashKind) - : directive - ? refChipElement(directive[1], directive[2]) - : null + recordUndoPoint?.() - if (chip) { - const space = document.createTextNode(' ') - const fragment = document.createDocumentFragment() - fragment.append(chip, space) - replaceRange.insertNode(fragment) + const fragment = document.createDocumentFragment() - const caret = document.createRange() - caret.setStart(space, 1) - caret.collapse(true) - sel.removeAllRanges() - sel.addRange(caret) + fragment.append(document.createTextNode(`@${next}`)) - return finish() + // In place first: the destructive re-render fallback rebuilds the editor + // from text, which is exactly what used to demote a leading command pill + // to plaintext on every Backspace inside a path. + if (!replaceBeforeCaret(editor, trigger.tokenLength, fragment)) { + rebuildAroundCaret(editor, trigger.tokenLength, `@${next}`) } - document.execCommand('insertText', false, text) - finish() + draftRef.current = composerPlainText(editor) + setComposerText(draftRef.current) + window.setTimeout(refreshTrigger, 0) + + return true } return { argStageEmpty, + ascendTriggerPath, closeTrigger, commitTypedSlashDirective, + moveTriggerActive, refreshTrigger, replaceTriggerWithChip, setTriggerActive, + slashFreeTextArgStage, trigger, triggerActive, + triggerActiveExplicit, triggerItems, triggerKeyConsumedRef, triggerLoading diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-undo.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-composer-undo.test.tsx new file mode 100644 index 000000000000..bbffc396ecb6 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-undo.test.tsx @@ -0,0 +1,193 @@ +import { render } from '@testing-library/react' +import { createRef, type RefObject } from 'react' +import { describe, expect, it, vi } from 'vitest' + +import { useComposerUndo } from './use-composer-undo' + +/** Mount the hook against a real contentEditable, exposing its API. */ +function mountUndo(editorRef: RefObject, onSync: () => string) { + const api: { current: ReturnType | null } = { current: null } + + const Harness = () => { + // Assigned during render on purpose: the tests drive the API imperatively + // right after mount, and this is a harness, not app state. + api.current = useComposerUndo({ editorRef, syncDraftFromEditor: onSync }) + + return null + } + + const view = render() + + return { api, view } +} + +function makeEditor(text: string) { + const editor = document.createElement('div') + editor.contentEditable = 'true' + // jsdom only focuses a contentEditable div when it's explicitly focusable; + // the real editor is reachable via the composer's focus bus. + editor.tabIndex = 0 + editor.append(document.createTextNode(text)) + document.body.append(editor) + + const ref = createRef() as RefObject + ref.current = editor + + return { editor, ref } +} + +const caretAtEnd = (editor: HTMLElement) => { + const range = document.createRange() + const selection = window.getSelection()! + range.selectNodeContents(editor) + range.collapse(false) + selection.removeAllRanges() + selection.addRange(range) +} + +describe('useComposerUndo', () => { + it('restores the pre-edit text, which is what a paste destroyed', () => { + const { editor, ref } = makeEditor('before') + caretAtEnd(editor) + + const { api, view } = mountUndo(ref, () => editor.textContent || '') + + // Bank, then simulate the Range-based paste that Chromium never records. + api.current!.recordUndoPoint() + editor.append(document.createTextNode(' PASTED')) + expect(editor.textContent).toBe('before PASTED') + + api.current!.undo() + expect(editor.textContent).toBe('before') + + api.current!.redo() + expect(editor.textContent).toBe('before PASTED') + + view.unmount() + editor.remove() + }) + + it('withUndoPoint banks only when the edit actually ran', () => { + const { editor, ref } = makeEditor('text') + caretAtEnd(editor) + + const { api, view } = mountUndo(ref, () => editor.textContent || '') + + // A guard that declines must not consume an undo slot. + expect(api.current!.withUndoPoint(() => false)).toBe(false) + expect(api.current!.undo()).toBe(false) + + expect( + api.current!.withUndoPoint(() => { + editor.append(document.createTextNode('!')) + + return true + }) + ).toBe(true) + + api.current!.undo() + expect(editor.textContent).toBe('text') + + view.unmount() + editor.remove() + }) + + it('claims a native historyUndo aimed at the focused editor', () => { + const { editor, ref } = makeEditor('kept') + editor.focus() + caretAtEnd(editor) + + const { api, view } = mountUndo(ref, () => editor.textContent || '') + + api.current!.recordUndoPoint() + editor.append(document.createTextNode(' extra')) + + // What Electron's Edit menu `{ role: 'undo' }` produces. + const event = new InputEvent('beforeinput', { bubbles: true, cancelable: true, inputType: 'historyUndo' }) + editor.dispatchEvent(event) + + expect(event.defaultPrevented).toBe(true) + expect(editor.textContent).toBe('kept') + + view.unmount() + editor.remove() + }) + + it('ignores a historyUndo while another editor holds focus', () => { + const { editor, ref } = makeEditor('mine') + const { editor: other } = makeEditor('theirs') + + other.focus() + + const { api, view } = mountUndo(ref, () => editor.textContent || '') + + api.current!.recordUndoPoint() + editor.append(document.createTextNode(' changed')) + + const event = new InputEvent('beforeinput', { bubbles: true, cancelable: true, inputType: 'historyUndo' }) + other.dispatchEvent(event) + + // Not ours to claim — the other surface keeps its native behavior. + expect(event.defaultPrevented).toBe(false) + expect(editor.textContent).toBe('mine changed') + + view.unmount() + editor.remove() + other.remove() + }) + + it('keeps two mounted composers independent', () => { + const { editor: main, ref: mainRef } = makeEditor('main') + const { editor: edit, ref: editRef } = makeEditor('edit') + + const mainUndo = mountUndo(mainRef, () => main.textContent || '') + const editUndo = mountUndo(editRef, () => edit.textContent || '') + + mainUndo.api.current!.recordUndoPoint() + main.append(document.createTextNode(' typed')) + + // Undoing in the edit composer must not touch the main composer's text. + editUndo.api.current!.undo() + expect(main.textContent).toBe('main typed') + + mainUndo.api.current!.undo() + expect(main.textContent).toBe('main') + expect(edit.textContent).toBe('edit') + + mainUndo.view.unmount() + editUndo.view.unmount() + main.remove() + edit.remove() + }) + + it('reset drops history so undo cannot cross a draft swap', () => { + const { editor, ref } = makeEditor('session A') + caretAtEnd(editor) + + const { api, view } = mountUndo(ref, () => editor.textContent || '') + + api.current!.recordUndoPoint() + editor.append(document.createTextNode(' edited')) + api.current!.resetUndoHistory() + + expect(api.current!.undo()).toBe(false) + expect(editor.textContent).toBe('session A edited') + + view.unmount() + editor.remove() + }) + + it('is inert when the editor ref is empty', () => { + const ref = createRef() as RefObject + const sync = vi.fn(() => '') + + const { api, view } = mountUndo(ref, sync) + + api.current!.recordUndoPoint() + + expect(api.current!.undo()).toBe(false) + expect(sync).not.toHaveBeenCalled() + + view.unmount() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-undo.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-undo.ts new file mode 100644 index 000000000000..72f891f6dac0 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-undo.ts @@ -0,0 +1,119 @@ +import { type RefObject, useCallback, useEffect, useMemo } from 'react' + +import { caretOffsetInEditor, composerPlainText, placeCaretAtOffset, renderComposerContents } from '../rich-editor' +import { type ComposerSnapshot, createComposerUndoHistory } from '../undo-history' + +interface UseComposerUndoArgs { + editorRef: RefObject + /** Push a restored snapshot back into draftRef + composer state. */ + syncDraftFromEditor: () => string +} + +/** + * Undo/redo for the rich composer. + * + * The editor mutates its DOM through `Range` to dodge Chromium's O(n²) editing + * pipeline (#45812), which also dodges Chromium's undo stack — so a paste was + * invisible to ⌘Z and the keystroke undid whatever edit came before it instead. + * We own the stack outright rather than half of it: every edit path records the + * pre-edit state here, and the editor claims ⌘Z / ⌘⇧Z itself. + */ +export function useComposerUndo({ editorRef, syncDraftFromEditor }: UseComposerUndoArgs) { + const history = useMemo(() => createComposerUndoHistory(), []) + + const snapshot = useCallback((): ComposerSnapshot => { + const editor = editorRef.current + + if (!editor) { + return { caret: 0, text: '' } + } + + return { caret: caretOffsetInEditor(editor), text: composerPlainText(editor) } + }, [editorRef]) + + /** Bank the current state before mutating the editor. `coalesce` marks a + * keystroke, so a run of typing collapses into one undo step. */ + const recordUndoPoint = useCallback( + (options?: { coalesce?: boolean }) => { + if (editorRef.current) { + history.record(snapshot(), options) + } + }, + [editorRef, history, snapshot] + ) + + const applySnapshot = useCallback( + (next: ComposerSnapshot | null) => { + const editor = editorRef.current + + if (!next || !editor) { + return false + } + + renderComposerContents(editor, next.text) + placeCaretAtOffset(editor, next.caret) + syncDraftFromEditor() + + return true + }, + [editorRef, syncDraftFromEditor] + ) + + /** Run a conditional edit, banking its pre-edit state only if it actually + * ran. The snapshot has to be taken first (the edit destroys the state we'd + * be saving), but recording unconditionally would clear the redo stack on + * every Backspace that falls through to the native path. */ + const withUndoPoint = useCallback( + (edit: () => boolean) => { + const before = snapshot() + const ran = edit() + + if (ran) { + history.record(before) + } + + return ran + }, + [history, snapshot] + ) + + const undo = useCallback(() => applySnapshot(history.undo(snapshot())), [applySnapshot, history, snapshot]) + const redo = useCallback(() => applySnapshot(history.redo(snapshot())), [applySnapshot, history, snapshot]) + + // A session/draft swap makes prior history meaningless — undoing into another + // conversation's text is worse than having no history at all. + const resetUndoHistory = useCallback(() => history.reset(), [history]) + + // Electron's Edit menu ships `{ role: 'undo' }`, whose accelerator the macOS + // menu bar consumes before the web contents sees the keystroke (the same + // hazard main.ts documents for ⌘W). It fires the native editing command, + // which knows nothing about our stack. Claim it at the document level while + // the composer holds focus, so the menu item and the keystroke agree. + useEffect(() => { + const onBeforeInput = (event: Event) => { + const inputType = (event as InputEvent).inputType + + if (inputType !== 'historyUndo' && inputType !== 'historyRedo') { + return + } + + if (document.activeElement !== editorRef.current) { + return + } + + event.preventDefault() + + if (inputType === 'historyUndo') { + undo() + } else { + redo() + } + } + + document.addEventListener('beforeinput', onBeforeInput, true) + + return () => document.removeEventListener('beforeinput', onBeforeInput, true) + }, [editorRef, redo, undo]) + + return { recordUndoPoint, redo, resetUndoHistory, undo, withUndoPoint } +} diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index f8b75183f30c..449471dc075c 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -1,15 +1,20 @@ +import { useStore } from '@nanostores/react' import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages' import { triggerHaptic } from '@/lib/haptics' +import { clearWakeIndicator, syncWakeIndicatorWithVoice } from '@/lib/wake-indicator' +import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' -import { notifyError } from '@/store/notifications' -import { $messages } from '@/store/session' -import { $autoSpeakReplies, setAutoSpeakReplies } from '@/store/voice-prefs' +import { $gateway } from '@/store/gateway' +import { notify, notifyError } from '@/store/notifications' +import { $autoSpeakReplies, $voiceStopPhrase, setAutoSpeakReplies } from '@/store/voice-prefs' +import { resumeWakeAfterVoice } from '@/store/wake-word' import type { ComposerTarget } from '../focus' import { onComposerVoiceToggleRequest } from '../focus' +import { useComposerScope } from '../scope' import type { ChatBarProps } from '../types' import { useAutoSpeakReplies } from './use-auto-speak-replies' @@ -23,6 +28,9 @@ interface UseComposerVoiceArgs { focusInput: () => void insertText: (text: string) => void maxRecordingSeconds: number + /** Interrupt the in-flight agent turn (Stop-button seam) — fired when the + * user speaks over the model while it is still generating. */ + onInterrupt?: () => Promise | void onSubmit: ChatBarProps['onSubmit'] onTranscribeAudio: ChatBarProps['onTranscribeAudio'] sessionId: string | null | undefined @@ -44,14 +52,19 @@ export function useComposerVoice({ focusInput, insertText, maxRecordingSeconds, + onInterrupt, onSubmit, onTranscribeAudio, sessionId, target }: UseComposerVoiceArgs) { const { t } = useI18n() + // A tile's composer speaks ITS transcript, not the primary chat's. + const { $messages } = useComposerScope() const [voiceConversationActive, setVoiceConversationActive] = useState(false) const lastSpokenIdRef = useRef(null) + const ownsWakeIndicatorRef = useRef(false) + const voiceStartRequest = useStore($voiceConversationStartRequest) const { dictate, voiceActivityState, voiceStatus } = useVoiceRecorder({ focusInput, @@ -109,16 +122,56 @@ export function useComposerVoice({ await onSubmit(text) } + const wakePausedRef = useRef(false) + // Resolves once the in-flight wake.pause round-trip completes (mic released by + // the wake listener). The conversation awaits this before opening its own mic + // so the two never contend for the device — on Windows especially, opening the + // capture device while the wake listener still holds it makes getUserMedia + // fail and the conversation never starts listening. + const wakePauseBarrierRef = useRef | null>(null) + const conversation = useVoiceConversation({ busy, consumePendingResponse, enabled: voiceConversationActive, onFatalError: () => setVoiceConversationActive(false), + // Speaking over the model mid-generation interrupts the in-flight turn — + // the same seam as the Stop button — so the interjection becomes the next + // turn instead of waiting behind a reply the user already rejected. + onInterrupt, + // A spoken stop command ("stop", "never mind", "goodbye", …) ends the + // hands-free conversation. Flipping the flag is the authoritative off + // switch — the enabled=false prop + effect below drive conversation.end() + // teardown (mic close, wake re-arm). + onStopWord: () => setVoiceConversationActive(false), onSubmit: submitVoiceTurn, onTranscribeAudio, - pendingResponse: pendingTurnResponse + pendingResponse: pendingTurnResponse, + // Before the conversation opens the mic, wait for any in-flight wake.pause + // to finish releasing the capture device (see wakePauseBarrierRef). + beforeMicOpen: () => wakePauseBarrierRef.current ?? undefined }) + // eslint-disable-next-line no-restricted-syntax -- ownership token used only by unmount cleanup + useEffect(() => { + if (target !== 'main') { + return + } + + if (syncWakeIndicatorWithVoice(voiceConversationActive, conversation.status)) { + ownsWakeIndicatorRef.current = voiceConversationActive + } + }, [conversation.status, target, voiceConversationActive]) + + useEffect( + () => () => { + if (ownsWakeIndicatorRef.current) { + clearWakeIndicator() + } + }, + [] + ) + // The `composer.voice` hotkey (Ctrl+B) toggles the conversation. Starting // with STT unconfigured lets the conversation surface its own "configure // speech-to-text" notice rather than silently no-opping. @@ -140,6 +193,73 @@ export function useComposerVoice({ [target, toggleVoiceConversation] ) + useEffect(() => { + if (target === 'main' && !disabled && takeVoiceConversationStart(voiceStartRequest) && !voiceConversationActive) { + setVoiceConversationActive(true) + } + }, [disabled, target, voiceConversationActive, voiceStartRequest]) + + const resumeWakeIfPaused = useCallback(() => { + if (!wakePausedRef.current) { + return + } + + wakePausedRef.current = false + wakePauseBarrierRef.current = null + // Reconcile, don't just resume: the wake word is a persistent setting, so + // ending a voice chat must re-arm the listener whenever config says + // enabled — including when the raw resume loses the mic-release race. + void resumeWakeAfterVoice() + }, []) + + // The ref is a request token (did WE issue wake.pause?), not an atom mirror — + // it guards resumeWakeIfPaused from resuming a detector another surface owns. + const pauseWakeForVoice = useCallback(() => { + wakePausedRef.current = true + + const barrier = (async () => { + try { + await $gateway.get()?.request('wake.pause', {}) + } catch { + // No wake listener / older backend — nothing held the mic. + } + })() + + wakePauseBarrierRef.current = barrier + + return barrier + }, []) + + useEffect(() => { + if (voiceConversationActive) { + pauseWakeForVoice() + } else { + resumeWakeIfPaused() + } + }, [pauseWakeForVoice, resumeWakeIfPaused, voiceConversationActive]) + + // 'Say "stop" to end the voice chat.' notice when the conversation starts. + // Phrase comes from voice.stop_phrases (first entry) so a custom phrase + // renders correctly; a null phrase (stop_phrases: []) shows no notice. + useEffect(() => { + if (!voiceConversationActive) { + return + } + + const phrase = $voiceStopPhrase.get() + + if (phrase) { + notify({ + id: 'voice-stop-hint', + kind: 'info', + icon: 'mic', + message: t.notifications.voice.sayStopToEnd(phrase) + }) + } + }, [t, voiceConversationActive]) + + useEffect(() => resumeWakeIfPaused, [resumeWakeIfPaused]) + // Explicit start/end for the on-screen conversation controls (the hotkey uses // the gated toggle above). const startConversation = useCallback(() => setVoiceConversationActive(true), []) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-emoji-completions.ts b/apps/desktop/src/app/chat/composer/hooks/use-emoji-completions.ts new file mode 100644 index 000000000000..0fb03526234f --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-emoji-completions.ts @@ -0,0 +1,122 @@ +import { useCallback } from 'react' + +import { type CompletionEntry, type CompletionPayload, useLiveCompletionAdapter } from './use-live-completion-adapter' + +/** + * `:shortcode:` completions for the composers, Slack-style (`:joy` → 😂). + * + * Draws from the same bundled emojibase-data the reaction picker uses (served + * at ./emojibase by the `hermes:emojibase-assets` vite plugin — offline, no + * CDN). The index lazy-loads on the first `:` trigger, then every query is + * answered from memory, so `isCached` skips the debounce and loading state + * after that first load. + * + * A pick inserts the emoji CHARACTER as plain text — not a chip. Directive + * chips exist to carry machine-readable references the backend resolves + * (@file:, /skill); a picked emoji is just text, so it rides the formatter's + * `rawText` path and lands inline. + */ + +interface EmojiEntry { + emoji: string + /** Primary shortcode, e.g. "joy". */ + code: string + /** Every shortcode, tag, and label that should match a search. */ + haystack: string[] +} + +let indexPromise: Promise | null = null +let indexLoaded = false + +async function loadIndex(): Promise { + const [dataRes, codesRes] = await Promise.all([ + fetch('./emojibase/en/data.json'), + fetch('./emojibase/en/shortcodes/emojibase.json') + ]) + + const data: { emoji: string; hexcode: string; label: string; tags?: string[] }[] = await dataRes.json() + const codes: Record = await codesRes.json() + const entries: EmojiEntry[] = [] + + for (const item of data) { + const raw = codes[item.hexcode] + + if (!raw) { + continue + } + + const shortcodes = Array.isArray(raw) ? raw : [raw] + + entries.push({ + emoji: item.emoji, + code: shortcodes[0], + haystack: [...shortcodes, ...(item.tags ?? []), item.label.toLowerCase()] + }) + } + + indexLoaded = true + + return entries +} + +/** Prefix matches on shortcodes rank first, then tag/label substring hits. */ +async function searchEmoji(query: string, limit = 8): Promise { + const index = await (indexPromise ??= loadIndex()) + const q = query.toLowerCase() + const prefix: EmojiEntry[] = [] + const loose: EmojiEntry[] = [] + + for (const entry of index) { + if (entry.code.startsWith(q) || entry.haystack.some(h => h.startsWith(q))) { + prefix.push(entry) + } else if (entry.haystack.some(h => h.includes(q))) { + loose.push(entry) + } + + if (prefix.length >= limit) { + break + } + } + + return [...prefix, ...loose].slice(0, limit) +} + +export function useEmojiCompletions() { + const fetcher = useCallback(async (query: string): Promise => { + const entries = await searchEmoji(query) + + return { + query, + items: entries.map(entry => ({ + text: entry.emoji, + display: `${entry.emoji} :${entry.code}:`, + meta: '' + })) + } + }, []) + + const toItem = useCallback( + (entry: CompletionEntry, index: number) => ({ + id: `${entry.text}|${index}`, + type: 'emoji', + label: typeof entry.display === 'string' ? entry.display : entry.text, + metadata: { + display: typeof entry.display === 'string' ? entry.display : entry.text, + // The formatter's serialize() returns rawText verbatim → the emoji + // character lands as plain inline text, no chip. + rawText: entry.text, + meta: '', + group: '', + action: '' + } + }), + [] + ) + + return useLiveCompletionAdapter({ + enabled: true, + fetcher, + isCached: () => indexLoaded, + toItem + }) +} diff --git a/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts b/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts index 6da699b602a1..d35c78ea7b3f 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-live-completion-adapter.ts @@ -25,9 +25,18 @@ export function useLiveCompletionAdapter(options: { enabled: boolean debounceMs?: number fetcher: (query: string) => Promise + /** True when `fetcher` will answer this query from cache. Such a query skips + * both the debounce and the loading state — the debounce exists to avoid a + * request per keystroke, and a spinner over an answer we already hold reads + * as latency the user isn't actually paying. */ + isCached?: (query: string) => boolean + /** Bump to declare the held answer stale. Without it a popover left open on + * an unchanged query would keep serving what it fetched before the source + * changed, because the adapter de-dupes on the query alone. */ + epoch?: number toItem: (entry: CompletionEntry, index: number) => Unstable_TriggerItem }): { adapter: Unstable_TriggerAdapter; loading: boolean } { - const { enabled, debounceMs = 60, fetcher, toItem } = options + const { enabled, debounceMs = 60, epoch = 0, fetcher, isCached, toItem } = options const [state, setState] = useState<{ query: string; items: Unstable_TriggerItem[] }>({ query: EMPTY_QUERY, @@ -49,6 +58,7 @@ export function useLiveCompletionAdapter(options: { useEffect(() => () => cancelTimer(), [cancelTimer]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (enabled) { return @@ -61,6 +71,16 @@ export function useLiveCompletionAdapter(options: { setState({ query: EMPTY_QUERY, items: [] }) }, [cancelTimer, enabled]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) + useEffect(() => { + // Invalidate by forgetting which query the held items answer, so the next + // search() re-fetches. The items themselves stay until the new answer + // lands — an open popover must not blink empty on a background refresh. + // On mount this is already the state, so the first run is a no-op. + pendingQueryRef.current = null + setState(current => (current.query === EMPTY_QUERY ? current : { ...current, query: EMPTY_QUERY })) + }, [epoch]) + const scheduleFetch = useCallback( (query: string) => { if (!enabled) { @@ -74,9 +94,13 @@ export function useLiveCompletionAdapter(options: { pendingQueryRef.current = query cancelTimer() const token = ++tokenRef.current - setLoading(true) + const cached = isCached?.(query) ?? false - timerRef.current = window.setTimeout(() => { + if (!cached) { + setLoading(true) + } + + const run = () => { timerRef.current = null fetcher(query) @@ -102,9 +126,13 @@ export function useLiveCompletionAdapter(options: { setLoading(false) } }) - }, debounceMs) + } + + // A cached answer resolves in a microtask, so debouncing it would only + // add a frame of empty popover on every keystroke. + cached ? run() : (timerRef.current = window.setTimeout(run, debounceMs)) }, - [cancelTimer, debounceMs, enabled, fetcher, toItem] + [cancelTimer, debounceMs, enabled, fetcher, isCached, toItem] ) const adapter = useMemo( diff --git a/apps/desktop/src/app/chat/composer/hooks/use-micro-actions.ts b/apps/desktop/src/app/chat/composer/hooks/use-micro-actions.ts new file mode 100644 index 000000000000..a8da94f04116 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-micro-actions.ts @@ -0,0 +1,47 @@ +import { useEffect } from 'react' + +import { useSessionSlice } from '@/lib/use-session-slice' +import { setComposerActions } from '@/store/composer-actions' +import { $todosBySession } from '@/store/todos' + +import { type ComposerMicroActionContext, useComposerMicroActionProviders } from '../contrib' + +/** + * Resolve every registered micro-action provider for this session and publish + * the result to `$composerActionsBySession`, which the pill strip renders. + * + * Core registers nothing, so the strip stays empty until something contributes + * to `composer.microActions`. Providers are pure functions of the session + * context and the set is recomputed rather than mutated, so there are no + * ordering games between registrars and a provider that stops returning a + * badge withdraws it. One that throws is skipped, so a broken plugin loses + * only its own badge. + */ +export function useComposerMicroActions(sessionId: null | string, busy: boolean) { + const todos = useSessionSlice($todosBySession, sessionId) + const providers = useComposerMicroActionProviders() + + useEffect(() => { + if (!sessionId) { + return + } + + const ctx: ComposerMicroActionContext = { busy, sessionId, todos } + + setComposerActions( + sessionId, + providers.flatMap(provider => { + try { + return provider.resolve(ctx) ?? [] + } catch { + return [] + } + }) + ) + }, [busy, providers, sessionId, todos]) + + // Withdraw on unmount / session switch ONLY. Clearing in the resolve effect's + // cleanup would publish an empty set before every republish — two store + // writes and two stack re-renders for what is usually a no-op. + useEffect(() => (sessionId ? () => setComposerActions(sessionId, []) : undefined), [sessionId]) +} diff --git a/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts b/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts index 0b71507bfd1a..e4a53889e5ba 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-popout-drag.ts @@ -3,6 +3,7 @@ import { type PointerEvent as ReactPointerEvent, type RefObject, useCallback, us import { POPOUT_ESTIMATED_HEIGHT, POPOUT_WIDTH_REM, + type PopoutBounds, type PopoutPosition, type PopoutSize, readPopoutBounds, @@ -35,6 +36,8 @@ interface PressState { interface ComposerPopoutGesturesOptions { composerRef: RefObject + /** Layout zone this composer belongs to — the scope its float is stored under. */ + groupId: string onDock: () => void onPopOut: () => void poppedOut: boolean @@ -67,10 +70,17 @@ function isFloatDragPlatform(target: EventTarget | null) { } /** 0 (far) → 1 (inside the dock zone). Drives both the dock glow and the - * release-to-dock test (which fires at proximity 1). */ -function dockProximityOf(rect: DOMRect) { - const horizontalDist = Math.abs(rect.left + rect.width / 2 - window.innerWidth / 2) - const verticalGap = window.innerHeight - DOCK_ZONE_BOTTOM_PX - rect.bottom + * release-to-dock test (which fires at proximity 1). + * + * Measured against THIS surface's area, not the window: the dock target is the + * docked composer, which sits at the bottom-center of its own chat surface. In + * a split (or any layout where the chat isn't the full window) the viewport's + * bottom-center is somewhere else entirely, so dragging onto the real dock + * never registered. */ +function dockProximityOf(rect: DOMRect, area?: PopoutBounds) { + const a = area ?? { bottom: window.innerHeight, left: 0, right: window.innerWidth, top: 0 } + const horizontalDist = Math.abs(rect.left + rect.width / 2 - (a.left + a.right) / 2) + const verticalGap = a.bottom - DOCK_ZONE_BOTTOM_PX - rect.bottom const v = verticalGap <= 0 ? 1 : Math.max(0, 1 - verticalGap / DOCK_VERTICAL_FALLOFF_PX) @@ -109,6 +119,7 @@ function popoutPositionUnderPointer( */ export function useComposerPopoutGestures({ composerRef, + groupId, onDock, onPopOut, poppedOut, @@ -142,7 +153,12 @@ export function useComposerPopoutGestures({ const beginFloatDrag = useCallback( (state: PressState, clientX: number, clientY: number, next: PopoutPosition, size?: PopoutSize) => { clearTimer() - const clamped = setComposerPopoutPosition(next, { area: readPopoutBounds(composerRef.current), size }) + + const clamped = setComposerPopoutPosition(groupId, next, { + area: readPopoutBounds(composerRef.current), + size + }) + liveRef.current = clamped state.mode = 'float' @@ -154,7 +170,7 @@ export function useComposerPopoutGestures({ setDragging(true) }, - [clearTimer, composerRef] + [clearTimer, composerRef, groupId] ) const peelOffFromDock = useCallback( @@ -231,6 +247,7 @@ export function useComposerPopoutGestures({ [clearTimer, poppedOut] ) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { // Coalesce drag updates to one per frame — pointermove can fire several times // between paints on high-Hz mice, and each update re-renders + clamps. @@ -254,17 +271,19 @@ export function useComposerPopoutGestures({ const composer = composerRef.current const size = composer ? { height: composer.offsetHeight, width: composer.offsetWidth } : undefined + const area = readPopoutBounds(composer) liveRef.current = setComposerPopoutPosition( + groupId, { bottom: state.startBottom - (pending.y - state.startY), right: state.startRight - (pending.x - state.startX) }, - { area: readPopoutBounds(composer), size } + { area, size } ) if (composer) { - setDockProximity(dockProximityOf(composer.getBoundingClientRect())) + setDockProximity(dockProximityOf(composer.getBoundingClientRect(), area)) } } @@ -316,13 +335,14 @@ export function useComposerPopoutGestures({ if (state.armed && state.mode === 'float') { const composer = composerRef.current const rect = composer?.getBoundingClientRect() + const area = readPopoutBounds(composer) - if (rect && dockProximityOf(rect) >= 1) { + if (rect && dockProximityOf(rect, area) >= 1) { onDock() } else { // Persist the resting position once, on release — never per move. const size = composer ? { height: composer.offsetHeight, width: composer.offsetWidth } : undefined - setComposerPopoutPosition(liveRef.current, { area: readPopoutBounds(composer), persist: true, size }) + setComposerPopoutPosition(groupId, liveRef.current, { area, persist: true, size }) } } @@ -339,7 +359,7 @@ export function useComposerPopoutGestures({ window.removeEventListener('pointerup', handleUp) window.removeEventListener('pointercancel', handleUp) } - }, [composerRef, onDock, peelOffFromDock, resetGesture]) + }, [composerRef, groupId, onDock, peelOffFromDock, resetGesture]) useEffect(() => clearTimer, [clearTimer]) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.test.tsx new file mode 100644 index 000000000000..a1e87ca53f26 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.test.tsx @@ -0,0 +1,157 @@ +import type { Unstable_TriggerItem } from '@assistant-ui/core' +import { act, cleanup, render } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import type { HermesGateway } from '@/hermes' +import { queryClient } from '@/lib/query-client' +import { invalidateSlashCompletions } from '@/lib/slash-completion-cache' + +import { isSkillItem } from '../composer-utils' + +import { useSlashCompletions } from './use-slash-completions' + +const CATALOG = { + categories: [{ name: 'Session', pairs: [['/new', 'Start a new session']] }], + pairs: [ + ['/new', 'Start a new session'], + ['/work', 'Kick off a task in a fresh worktree'] + ] +} + +// A catalog shaped like a real install: a couple of skills the user lives in, +// a bundled one they have never opened, and one of their own they haven't +// either. +const RANKED_CATALOG = { + categories: [{ name: 'Session', pairs: [['/new', 'Start a new session']] }], + pairs: [ + ['/new', 'Start a new session'], + ['/docx', 'Edit Word documents'], + ['/research', 'Look it up before answering'], + ['/research-paper-writing', 'Write an academic paper'], + ['/work', 'Kick off a task in a fresh worktree'] + ], + skills: { + '/docx': { usage: 0, origin: 'local' }, + '/research': { usage: 60, origin: 'local' }, + '/research-paper-writing': { usage: 0, origin: 'bundled' }, + '/work': { usage: 172, origin: 'local' } + } +} + +const commandsOf = (items: readonly Unstable_TriggerItem[]) => + items.map(item => (item.metadata as { command?: string })?.command) + +function harness(gateway: HermesGateway) { + const api: { search?: (query: string) => readonly Unstable_TriggerItem[] } = {} + + function Probe() { + const { adapter } = useSlashCompletions({ gateway }) + api.search = adapter.search + + return null + } + + render() + + return api as { search: (query: string) => readonly Unstable_TriggerItem[] } +} + +/** Drive the adapter until its async fetch has settled into `search`'s result. */ +async function completions(api: { search: (query: string) => readonly Unstable_TriggerItem[] }, query: string) { + await act(async () => { + api.search(query) + await Promise.resolve() + }) + + // The debounce is skipped only for cached queries; give the timer a beat. + await act(async () => { + await new Promise(resolve => setTimeout(resolve, 120)) + }) + + return api.search(query) +} + +afterEach(() => { + cleanup() + queryClient.clear() +}) + +describe('useSlashCompletions', () => { + it('serves the bare-slash catalog from cache instead of re-requesting it', async () => { + const request = vi.fn().mockResolvedValue(CATALOG) + const api = harness({ request } as unknown as HermesGateway) + + await completions(api, '') + expect(request).toHaveBeenCalledTimes(1) + + // Reopening `/` must not hit the gateway again. + queryClient.setQueryData(['unrelated'], 1) + await completions(api, '') + expect(request).toHaveBeenCalledTimes(1) + + // …until something that changes the command set invalidates it. + await act(async () => invalidateSlashCompletions()) + await completions(api, '') + expect(request).toHaveBeenCalledTimes(2) + }) + + it('offers skill commands on a bare slash, not just built-ins', async () => { + const request = vi.fn().mockResolvedValue(CATALOG) + const api = harness({ request } as unknown as HermesGateway) + + const items = await completions(api, '') + const work = items.find(item => (item.metadata as { command?: string })?.command === '/work') + + expect((work?.metadata as { group?: string })?.group).toBe('Skills') + }) + + // A `/` typed mid-message is a reference dropped into prose, so the trigger + // filters the list to skills (use-composer-trigger). A bare mid-message `/` + // resolves to the same empty query as an opening `/`, so that filter runs + // over the catalog — which listed no skill-group rows at all, leaving the + // inline popover empty. Asserted through isSkillItem, the real predicate. + it('leaves only skills for a mid-message slash', async () => { + const request = vi.fn().mockResolvedValue(CATALOG) + const api = harness({ request } as unknown as HermesGateway) + + const inline = (await completions(api, '')).filter(isSkillItem) + + expect(inline.map(item => (item.metadata as { command?: string })?.command)).toEqual(['/work']) + }) + + // An alphabetical `/` menu buries the skills someone runs daily under the + // ones that shipped with Hermes and were never opened. + it('orders skills by use and hides never-used built-ins on a bare slash', async () => { + const request = vi.fn().mockResolvedValue(RANKED_CATALOG) + const api = harness({ request } as unknown as HermesGateway) + + const skills = commandsOf((await completions(api, '')).filter(isSkillItem)) + + expect(skills).toEqual(['/work', '/research', '/docx']) + }) + + // Typing is a search, and a search that hides a match is broken — the + // never-used built-in still shows, just below the one she actually uses. + it('ranks a typed query by use without hiding anything', async () => { + const request = vi.fn().mockImplementation((method: string) => + Promise.resolve( + method === 'commands.catalog' + ? RANKED_CATALOG + : { + items: [ + { text: '/research-paper-writing', display: '/research-paper-writing', meta: 'Write a paper' }, + { text: '/research', display: '/research', meta: 'Look it up' } + ] + } + ) + ) + + const api = harness({ request } as unknown as HermesGateway) + + // Warm the catalog first: the popover always opens on a bare `/` before a + // query is typed, which is where the usage map comes from. + await completions(api, '') + + expect(commandsOf(await completions(api, 'research'))).toEqual(['/research', '/research-paper-writing']) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.ts b/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.ts index bf6e5006beae..32f0038fe2e5 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-slash-completions.ts @@ -1,4 +1,5 @@ import type { Unstable_TriggerAdapter, Unstable_TriggerItem } from '@assistant-ui/core' +import { useStore } from '@nanostores/react' import { useCallback } from 'react' import type { HermesGateway } from '@/hermes' @@ -10,8 +11,15 @@ import { type DesktopThemeCommandOption, filterDesktopCommandsCatalog, isDesktopSlashExtensionCommand, - isDesktopSlashSuggestion + isDesktopSlashSuggestion, + rankSkillCommands } from '@/lib/desktop-slash-commands' +import { + $slashCompletionsEpoch, + cachedSlashCompletion, + hasCachedSlashCompletion, + peekCachedSlashCompletion +} from '@/lib/slash-completion-cache' import { normalize } from '@/lib/text' import { $sessions } from '@/store/session' @@ -63,6 +71,7 @@ export function useSlashCompletions(options: { } { const { gateway, skinThemes, activeSkin } = options const enabled = Boolean(gateway) + const epoch = useStore($slashCompletionsEpoch) const fetcher = useCallback( async (query: string): Promise => { @@ -133,14 +142,16 @@ export function useSlashCompletions(options: { try { if (!query) { - const catalog = filterDesktopCommandsCatalog(await gateway.request('commands.catalog')) + const catalog = filterDesktopCommandsCatalog( + await cachedSlashCompletion('catalog', () => gateway.request('commands.catalog')) + ) // Prefer the categorized layout so the popover renders section headers // (Session, Tools & Skills, ...). Fall back to the flat list when the // backend didn't categorize. const sections = catalog.categories?.length ? catalog.categories : [{ name: '', pairs: catalog.pairs ?? [] }] - const items = sections.flatMap(section => + const items = sections.flatMap(section => section.pairs.map(([command, meta]) => ({ text: command, display: command, @@ -149,12 +160,32 @@ export function useSlashCompletions(options: { })) ) + // Skill commands reach us only through the flat `pairs` list — the + // backend categorizes registry commands but appends skills + // uncategorized, so the categorized layout alone drops every skill + // from the bare `/` list even though typing `/wo` offers them. + // Re-add the leftovers under one Skills header (which also gives them + // the skill pill accent and makes them offerable mid-message). + const categorized = new Set(items.map(item => item.text.toLowerCase())) + const skillRows: CompletionEntry[] = [] + + for (const [command, meta] of catalog.pairs ?? []) { + if (!categorized.has(command.toLowerCase()) && isDesktopSlashExtensionCommand(command)) { + skillRows.push({ text: command, display: command, group: 'Skills', meta }) + } + } + + // Browsing, not searching: rank the skills the user actually reaches + // for to the top and drop never-used built-ins entirely. Typing a + // query takes the other branch, where nothing is hidden. + items.push(...rankSkillCommands(skillRows, catalog.skills, { pruneUnusedBuiltins: true })) + return { items, query } } - const result = await gateway.request<{ items?: CompletionEntry[]; replace_from?: number }>('complete.slash', { - text - }) + const result = await cachedSlashCompletion(`slash:${text.toLowerCase()}`, () => + gateway.request<{ items?: CompletionEntry[]; replace_from?: number }>('complete.slash', { text }) + ) // Arg-completion items (replace_from > 1) carry just the arg stub — // e.g. complete.slash returns `{text: "alice"}` for `/personality alic` @@ -191,9 +222,27 @@ export function useSlashCompletions(options: { // Skills (stable within a group, preserving backend relevance order). const groupOrder = ['Commands', 'Skills', 'Options'] - const items = isArgCompletion - ? decorated - : [...decorated].sort((a, b) => groupOrder.indexOf(a.group) - groupOrder.indexOf(b.group)) + if (isArgCompletion) { + return { items: decorated, query } + } + + // Rank the matched skills by use — `/re` should lead with the /research + // the user lives in, not the /research-paper-writing they've never + // opened. Nothing is pruned here: a typed query is a search, and a + // search that hides a match is broken. Usage rides along on the catalog + // response, which the popover has already fetched by the time anyone + // types; if it somehow hasn't, order falls back to the backend's. + const catalogSkills = peekCachedSlashCompletion('catalog')?.skills + + const ranked = [ + ...decorated.filter(item => item.group !== 'Skills'), + ...rankSkillCommands( + decorated.filter(item => item.group === 'Skills'), + catalogSkills + ) + ] + + const items = [...ranked].sort((a, b) => groupOrder.indexOf(a.group) - groupOrder.indexOf(b.group)) return { items, query } } catch { @@ -231,5 +280,21 @@ export function useSlashCompletions(options: { } }, []) - return useLiveCompletionAdapter({ enabled, fetcher, toItem }) + // Mirrors the fetcher's branching: the `/skin` and `/resume` arg stages are + // answered from client-side state, so they never wait on the network; every + // other query is served from the completion cache when it's still warm. + const isCached = useCallback( + (query: string) => { + const text = `/${query}` + + if ((skinThemes && /^\/skin\s+/is.test(text)) || /^\/(?:resume|sessions|switch)\s+/is.test(text)) { + return true + } + + return hasCachedSlashCompletion(query ? `slash:${text.toLowerCase()}` : 'catalog') + }, + [skinThemes] + ) + + return useLiveCompletionAdapter({ enabled, epoch, fetcher, isCached, toItem }) } diff --git a/apps/desktop/src/app/chat/composer/hooks/use-status-presence.ts b/apps/desktop/src/app/chat/composer/hooks/use-status-presence.ts index c6b9af53b737..b4655ffd50cb 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-status-presence.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-status-presence.ts @@ -1,26 +1,37 @@ import { useSyncExternalStore } from 'react' +import { $composerActionsBySession } from '@/store/composer-actions' import { $statusItemsBySession } from '@/store/composer-status' import { $previewStatusBySession } from '@/store/preview-status' +/** Structural view of the three per-session feeds — they hold different item + * types, and all this hook needs from each is "does this key have rows". */ +interface PresenceFeed { + get(): Record + listen(listener: () => void): () => void +} + +const FEEDS: PresenceFeed[] = [$statusItemsBySession, $composerActionsBySession, $previewStatusBySession] + const subscribe = (onChange: () => void) => { - const offItems = $statusItemsBySession.listen(onChange) - const offPreviews = $previewStatusBySession.listen(onChange) + const offs = FEEDS.map(feed => feed.listen(onChange)) return () => { - offItems() - offPreviews() + for (const off of offs) { + off() + } } } /** - * Whether a session has any status items or previews, as a coarse *edge*: the - * boolean only flips when the stack appears/disappears. ChatBar uses it to - * toggle a styling data-attr — subscribing to the whole `$statusItemsBySession` - * (a `computed` that rebuilds the entire map) / `$previewStatusBySession` maps - * re-rendered the ~1.4k ChatBar on every per-item mutation (a subagent tick, a - * 5s background poll) and on churn in OTHER sessions. The boolean snapshot bails - * out of all of that, re-rendering only on the actual show/hide transition. + * Whether a session has any status items, micro actions, or previews, as a + * coarse *edge*: the boolean only flips when the stack appears/disappears. + * ChatBar uses it to toggle a styling data-attr — subscribing to the whole + * `$statusItemsBySession` (a `computed` that rebuilds the entire map) / + * `$previewStatusBySession` maps re-rendered the ~1.4k ChatBar on every + * per-item mutation (a subagent tick, a 5s background poll) and on churn in + * OTHER sessions. The boolean snapshot bails out of all of that, re-rendering + * only on the actual show/hide transition. */ export function useSessionStatusPresence(sessionId: string | null): boolean { return useSyncExternalStore(subscribe, () => { @@ -28,9 +39,6 @@ export function useSessionStatusPresence(sessionId: string | null): boolean { return false } - return ( - ($statusItemsBySession.get()[sessionId]?.length ?? 0) > 0 || - ($previewStatusBySession.get()[sessionId]?.length ?? 0) > 0 - ) + return FEEDS.some(feed => (feed.get()[sessionId]?.length ?? 0) > 0) }) } diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-rearm.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-rearm.test.tsx new file mode 100644 index 000000000000..e993abc57309 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation-rearm.test.tsx @@ -0,0 +1,258 @@ +import { act, cleanup, renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { $voicePlayback } from '@/store/voice-playback' + +import { useVoiceConversation } from './use-voice-conversation' + +const mocks = vi.hoisted(() => { + let deferStreamStart = false + let onSilence: null | (() => void) = null + let resolveStreamStart: null | (() => void) = null + let resolveSpeech: null | ((outcome: 'done' | 'fallback') => void) = null + let streamAvailable = true + + const stopVoicePlayback = vi.fn(() => { + const current = $voicePlayback.get() + $voicePlayback.set({ ...current, sequence: current.sequence + 1, status: 'idle' }) + }) + + const playSpeechText = vi.fn(() => { + stopVoicePlayback() + + return Promise.resolve(true) + }) + + const handle = { + cancel: vi.fn(), + start: vi.fn(async (options: { onSilence: () => void }) => { + onSilence = options.onSilence + }), + stop: vi.fn(async () => ({ + audio: new Blob(['voice'], { type: 'audio/webm' }), + heardSpeech: true + })) + } + + return { + continueStreamStart() { + resolveStreamStart?.() + resolveStreamStart = null + }, + deferStreamStart() { + deferStreamStart = true + }, + finishSpeech(outcome: 'done' | 'fallback') { + resolveSpeech?.(outcome) + }, + handle, + playSpeechText, + resetSpeechMocks() { + deferStreamStart = false + resolveStreamStart = null + resolveSpeech = null + streamAvailable = true + }, + startSpeechStream: vi.fn(async () => { + if (deferStreamStart) { + await new Promise(resolve => { + resolveStreamStart = resolve + }) + } + + if (!streamAvailable) { + return null + } + + const current = $voicePlayback.get() + $voicePlayback.set({ ...current, sequence: current.sequence + 1, status: 'preparing' }) + + return { + append: vi.fn(), + done: new Promise<'done' | 'fallback'>(resolve => { + resolveSpeech = resolve + }), + finish: vi.fn() + } + }), + stopVoicePlayback, + triggerSilence() { + onSilence?.() + }, + useFallbackSpeech() { + streamAvailable = false + } + } +}) + +vi.mock('./use-mic-recorder', () => ({ + useMicRecorder: () => ({ handle: mocks.handle, level: 0 }) +})) + +vi.mock('@/lib/voice-barge-in', () => ({ + monitorSpeechDuringPlayback: () => vi.fn() +})) + +vi.mock('@/lib/voice-playback', () => ({ + markVoicePlaybackInterrupted: vi.fn(), + playSpeechText: mocks.playSpeechText, + startSpeechStream: mocks.startSpeechStream, + stopVoicePlayback: mocks.stopVoicePlayback +})) + +vi.mock('@/lib/thinking-sound', () => ({ + startThinkingSound: vi.fn(), + stopThinkingSound: vi.fn() +})) + +vi.mock('@/store/notifications', () => ({ + notify: vi.fn(), + notifyError: vi.fn() +})) + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + notifications: { + voice: { + configureSpeechToText: '', + couldNotStartSession: '', + microphoneFailed: '', + playbackFailed: '', + transcriptionFailed: '', + unavailable: '' + } + } + } + }) +})) + +function renderRearmConversation(responseId: string, responseText: string) { + let response: null | { id: string; pending: boolean; text: string } = null + + return renderHook( + ({ enabled }) => + useVoiceConversation({ + busy: false, + consumePendingResponse: vi.fn(), + enabled, + onSubmit: async () => { + response = { id: responseId, pending: false, text: responseText } + }, + onTranscribeAudio: async () => 'Hello', + pendingResponse: () => response + }), + { initialProps: { enabled: false } } + ) +} + +async function beginReply(hook: ReturnType) { + hook.rerender({ enabled: true }) + await waitFor(() => expect(mocks.handle.start).toHaveBeenCalledTimes(1)) + + await act(async () => { + mocks.triggerSilence() + }) +} + +describe('useVoiceConversation playback rearm', () => { + afterEach(() => { + cleanup() + vi.clearAllMocks() + mocks.resetSpeechMocks() + $voicePlayback.set({ + audioElement: null, + messageId: null, + sequence: 0, + source: null, + status: 'idle' + }) + }) + + it('re-arms the microphone after normal streaming playback completes', async () => { + $voicePlayback.set({ + audioElement: null, + messageId: null, + sequence: 7, + source: null, + status: 'idle' + }) + const hook = renderRearmConversation('reply-1', 'Hello back') + + await beginReply(hook) + await waitFor(() => expect(mocks.startSpeechStream).toHaveBeenCalled()) + expect($voicePlayback.get().sequence).toBeGreaterThan(7) + + await act(async () => { + mocks.finishSpeech('done') + }) + + await waitFor(() => expect(mocks.handle.start).toHaveBeenCalledTimes(2)) + expect(hook.result.current.status).toBe('listening') + }) + + it('honors Stop while streaming playback is still preparing', async () => { + mocks.deferStreamStart() + const hook = renderRearmConversation('reply-preparing', 'Do not play this') + + await beginReply(hook) + await waitFor(() => expect(mocks.startSpeechStream).toHaveBeenCalled()) + + mocks.stopVoicePlayback() + await act(async () => { + mocks.continueStreamStart() + }) + + await waitFor(() => expect(hook.result.current.status).toBe('idle')) + expect(mocks.stopVoicePlayback).toHaveBeenCalledTimes(2) + expect(mocks.handle.start).toHaveBeenCalledTimes(1) + }) + + it('does not start fallback playback after Stop during stream discovery', async () => { + mocks.deferStreamStart() + mocks.useFallbackSpeech() + const hook = renderRearmConversation('reply-no-stream', 'Do not fall back') + + await beginReply(hook) + await waitFor(() => expect(mocks.startSpeechStream).toHaveBeenCalled()) + + mocks.stopVoicePlayback() + await act(async () => { + mocks.continueStreamStart() + }) + + await waitFor(() => expect(hook.result.current.status).toBe('idle')) + expect(mocks.playSpeechText).not.toHaveBeenCalled() + expect(mocks.handle.start).toHaveBeenCalledTimes(1) + }) + + it('does not re-arm after an external Stop during streaming playback', async () => { + const hook = renderRearmConversation('reply-stopped', 'Playing now') + + await beginReply(hook) + await waitFor(() => expect(mocks.startSpeechStream).toHaveBeenCalled()) + + mocks.stopVoicePlayback() + await act(async () => { + mocks.finishSpeech('done') + }) + + await waitFor(() => expect(hook.result.current.status).toBe('idle')) + expect(mocks.handle.start).toHaveBeenCalledTimes(1) + }) + + it('re-arms the microphone after normal fallback playback completes', async () => { + mocks.useFallbackSpeech() + const hook = renderRearmConversation('reply-fallback', 'Fallback reply') + + await beginReply(hook) + + await waitFor(() => + expect(mocks.playSpeechText).toHaveBeenCalledWith('Fallback reply', { + source: 'voice-conversation' + }) + ) + await waitFor(() => expect(mocks.handle.start).toHaveBeenCalledTimes(2)) + expect(hook.result.current.status).toBe('listening') + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx new file mode 100644 index 000000000000..1e43ef8b1863 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx @@ -0,0 +1,266 @@ +import { act, cleanup, renderHook, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { BargeMonitorCallbacks } from '@/lib/voice-barge-in' + +import type { MicRecording } from './use-mic-recorder' +import { useVoiceConversation } from './use-voice-conversation' + +// The full-duplex contract: the barge monitor is live across the WHOLE agent +// turn — generation (thinking) and playback (speaking) — so speaking over the +// model interrupts it mid-generation instead of the mic being deaf until TTS +// starts (the Windows report: interruption "never works" because the deaf +// window covered generation, and playback bleed made the old monitor's +// trigger unreachable). + +const monitorCalls: BargeMonitorCallbacks[] = [] +const stopMonitor = vi.fn() + +vi.mock('@/lib/voice-barge-in', () => ({ + monitorSpeechDuringPlayback: (callbacks: BargeMonitorCallbacks) => { + monitorCalls.push(callbacks) + + return stopMonitor + } +})) + +const markVoicePlaybackInterrupted = vi.fn() +const stopVoicePlayback = vi.fn() + +vi.mock('@/lib/voice-playback', () => ({ + markVoicePlaybackInterrupted: () => markVoicePlaybackInterrupted(), + playSpeechText: vi.fn(async () => true), + startSpeechStream: vi.fn(async () => null), + stopVoicePlayback: () => stopVoicePlayback() +})) + +vi.mock('@/lib/thinking-sound', () => ({ + startThinkingSound: vi.fn(), + stopThinkingSound: vi.fn() +})) + +const micHandle = { + cancel: vi.fn(), + start: vi.fn(async () => undefined), + stop: vi.fn<() => Promise>(async () => null) +} + +vi.mock('./use-mic-recorder', () => ({ + useMicRecorder: () => ({ handle: micHandle, level: 0, recording: false }) +})) + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + notifications: { + voice: { + configureSpeechToText: 'configure STT', + couldNotStartSession: 'could not start', + microphoneFailed: 'mic failed', + playbackFailed: 'playback failed', + transcriptionFailed: 'transcription failed', + unavailable: 'unavailable' + } + } + } + }) +})) + +vi.mock('@/store/notifications', () => ({ + notify: vi.fn(), + notifyError: vi.fn() +})) + +interface HookProps { + busy: boolean +} + +function renderConversation(overrides: { onInterrupt?: () => void; transcript?: string } = {}) { + const onInterrupt = overrides.onInterrupt ?? vi.fn() + + // Mirrors the real app: submitting a turn makes the agent busy. + const onBusyChange: { current: (busy: boolean) => void } = { current: () => undefined } + + const onSubmit = vi.fn(async () => { + onBusyChange.current(true) + }) + + const onStopWord = vi.fn() + + // First transcription is the turn that starts the conversation; subsequent + // ones are barge captures (the overridable transcript). + let transcriptions = 0 + + const onTranscribeAudio = vi.fn(async () => + transcriptions++ === 0 ? 'kick off the task' : (overrides.transcript ?? 'and another thing') + ) + + const hook = renderHook( + ({ busy }: HookProps) => + useVoiceConversation({ + busy, + consumePendingResponse: vi.fn(), + enabled: true, + onInterrupt, + onStopWord, + onSubmit, + onTranscribeAudio, + pendingResponse: () => null + }), + { initialProps: { busy: false } } + ) + + onBusyChange.current = busy => hook.rerender({ busy }) + + return { hook, onInterrupt, onStopWord, onSubmit, onTranscribeAudio } +} + +/** Drive the hook into the generation phase (turn submitted, model working). */ +async function enterThinking(hook: ReturnType['hook']) { + await act(async () => { + await hook.result.current.start() + }) + await waitFor(() => expect(hook.result.current.status).toBe('listening')) + + micHandle.stop.mockResolvedValueOnce({ + audio: new Blob(['q'], { type: 'audio/webm' }), + durationMs: 900, + heardSpeech: true + }) + + await act(async () => { + hook.result.current.stopTurn() + }) + await waitFor(() => expect(hook.result.current.status).toBe('thinking')) +} + +describe('useVoiceConversation full-duplex barge-in', () => { + beforeEach(() => { + monitorCalls.length = 0 + vi.clearAllMocks() + micHandle.start.mockResolvedValue(undefined) + micHandle.stop.mockResolvedValue(null) + }) + + afterEach(cleanup) + + it('arms the barge monitor during generation (before any reply audio exists)', async () => { + const { hook } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + + await waitFor(() => expect(hook.result.current.status).toBe('thinking')) + // busy=true + thinking → the full-duplex monitor must be live. + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + }) + + it('interrupts the in-flight turn when speech trips mid-generation', async () => { + const { hook, onInterrupt } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + act(() => { + monitorCalls.at(-1)?.onSpeech() + }) + + expect(onInterrupt).toHaveBeenCalledTimes(1) + expect(markVoicePlaybackInterrupted).toHaveBeenCalled() + expect(stopVoicePlayback).toHaveBeenCalled() + }) + + it('submits the captured interruption once the interrupt settles (busy clears)', async () => { + const { hook, onSubmit } = renderConversation({ transcript: 'no, do it differently' }) + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + const monitor = monitorCalls.at(-1) + + act(() => { + monitor?.onSpeech() + }) + + // Interrupt lands → the turn ends → busy flips false. + hook.rerender({ busy: false }) + + await act(async () => { + monitor?.onUtterance?.(new Blob(['x'], { type: 'audio/webm' })) + }) + + await waitFor(() => expect(onSubmit).toHaveBeenCalledWith('no, do it differently')) + }) + + it('does not interrupt when speech trips during playback (turn already done)', async () => { + const { hook, onInterrupt } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + // Turn finished; playback phase. + hook.rerender({ busy: false }) + + act(() => { + monitorCalls.at(-1)?.onSpeech() + }) + + expect(onInterrupt).not.toHaveBeenCalled() + expect(stopVoicePlayback).toHaveBeenCalled() + }) + + it('a spoken stop command in the barge capture ends the conversation instead of submitting', async () => { + const { hook, onStopWord, onSubmit } = renderConversation({ transcript: 'stop' }) + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + const monitor = monitorCalls.at(-1) + + act(() => { + monitor?.onSpeech() + }) + hook.rerender({ busy: false }) + + await act(async () => { + monitor?.onUtterance?.(new Blob(['s'], { type: 'audio/webm' })) + }) + + await waitFor(() => expect(onStopWord).toHaveBeenCalledTimes(1)) + // Only the kickoff turn was submitted — the "stop" capture never was. + expect(onSubmit).toHaveBeenCalledTimes(1) + expect(onSubmit).not.toHaveBeenCalledWith('stop') + }) + + it('re-arms a single monitor per turn (idempotent ensure)', async () => { + const { hook } = renderConversation() + + await act(async () => { + await hook.result.current.start() + }) + await enterThinking(hook) + await waitFor(() => expect(monitorCalls.length).toBeGreaterThan(0)) + + const armed = monitorCalls.length + + // Effect re-runs (busy toggles, status changes) must not open more mics. + hook.rerender({ busy: true }) + hook.rerender({ busy: true }) + + expect(monitorCalls.length).toBe(armed) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index 2ec43c83c0b8..55978c428474 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -1,6 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' +import { startThinkingSound, stopThinkingSound } from '@/lib/thinking-sound' import { monitorSpeechDuringPlayback } from '@/lib/voice-barge-in' import { markVoicePlaybackInterrupted, @@ -9,7 +10,9 @@ import { startSpeechStream, stopVoicePlayback } from '@/lib/voice-playback' +import { isVoiceStopCommand } from '@/lib/voice-stop-word' import { notify, notifyError } from '@/store/notifications' +import { $voicePlayback } from '@/store/voice-playback' import { useMicRecorder } from './use-mic-recorder' @@ -25,20 +28,34 @@ interface VoiceConversationOptions { busy: boolean enabled: boolean onFatalError?: () => void + /** Interrupt the in-flight agent turn (the same seam as the Stop button). + * Fired when the user speaks while the model is still generating. */ + onInterrupt?: () => Promise | void + onStopWord?: () => void onSubmit: (text: string) => Promise | void onTranscribeAudio?: (audio: Blob) => Promise pendingResponse: () => PendingVoiceResponse | null consumePendingResponse: () => void + /** Awaited right before the mic is opened. Used to let the wake-word listener + * fully release the capture device first, so the two never contend. */ + beforeMicOpen?: () => Promise | void } +/** How long a barge-triggered interrupt may take to settle before we submit + * the captured utterance anyway. */ +const INTERRUPT_SETTLE_TIMEOUT_MS = 5_000 + export function useVoiceConversation({ busy, enabled, onFatalError, + onInterrupt, + onStopWord, onSubmit, onTranscribeAudio, pendingResponse, - consumePendingResponse + consumePendingResponse, + beforeMicOpen }: VoiceConversationOptions) { const { t } = useI18n() const voiceCopy = t.notifications.voice @@ -54,24 +71,49 @@ export function useVoiceConversation({ const speechSessionRef = useRef(null) const stopBargeMonitorRef = useRef<(() => void) | null>(null) const bargeCapturePendingRef = useRef(false) + const bargedRef = useRef(false) + const speechStartSequenceRef = useRef(0) const enabledRef = useRef(enabled) const mutedRef = useRef(muted) const busyRef = useRef(busy) const statusRef = useRef('idle') const wasEnabledRef = useRef(enabled) + const onStopWordRef = useRef(onStopWord) + const onInterruptRef = useRef(onInterrupt) + + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) + useEffect(() => { + onInterruptRef.current = onInterrupt + }, [onInterrupt]) + + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) + useEffect(() => { + onStopWordRef.current = onStopWord + }, [onStopWord]) + + const beforeMicOpenRef = useRef(beforeMicOpen) + + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) + useEffect(() => { + beforeMicOpenRef.current = beforeMicOpen + }, [beforeMicOpen]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { enabledRef.current = enabled }, [enabled]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { mutedRef.current = muted }, [muted]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { busyRef.current = busy }, [busy]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { statusRef.current = status }, [status]) @@ -87,6 +129,7 @@ export function useVoiceConversation({ stopBargeMonitorRef.current?.() stopBargeMonitorRef.current = null bargeCapturePendingRef.current = false + bargedRef.current = false speechSessionRef.current = null responseIdRef.current = null spokenSourceLengthRef.current = 0 @@ -128,6 +171,18 @@ export function useVoiceConversation({ return } + // A spoken "stop" (or "never mind", "goodbye", …) ends the + // conversation instead of being submitted as a turn. Only whole- + // utterance stop commands match, so "stop the container" still goes + // through as a real request. + if (isVoiceStopCommand(transcript)) { + dropSpeechSession() + setStatus('idle') + onStopWordRef.current?.() + + return + } + awaitingSpokenResponseRef.current = true dropSpeechSession() await onSubmit(transcript) @@ -163,6 +218,20 @@ export function useVoiceConversation({ return } + // Let the wake-word listener fully release the capture device before we + // open ours — opening the mic while wake still holds it makes getUserMedia + // fail (the "clicked voice but it never starts listening" bug). + try { + await beforeMicOpenRef.current?.() + } catch { + // A pause failure shouldn't block the user's explicit start. + } + + // enabled/muted/busy or an interleaved turn may have changed while we waited. + if (!enabledRef.current || mutedRef.current || busyRef.current || statusRef.current !== 'idle') { + return + } + try { // VAD tuning mirrors `tools.voice_mode` defaults so the browser loop matches the CLI. await handle.start({ @@ -177,6 +246,12 @@ export function useVoiceConversation({ onSilence: () => void handleTurn() }) setStatus('listening') + // Clear any prior turn-timeout before arming a fresh one. Each listen + // cycle reassigns turnTimeoutRef; without clearing first, a stale 60s + // timer from an earlier cycle survives and later fires handleTurn() in + // the middle of a new listen, cutting it short (or, after enough idle + // re-listens, wedging the loop into a state it doesn't re-arm from). + clearTurnTimeout() turnTimeoutRef.current = window.setTimeout(() => void handleTurn(), 60_000) } catch (error) { notifyError(error, voiceCopy.couldNotStartSession) @@ -187,7 +262,7 @@ export function useVoiceConversation({ }, [handle, handleTurn, onFatalError, voiceCopy.couldNotStartSession, voiceCopy.microphoneFailed]) const settleAfterSpeech = useCallback( - (barged: boolean) => { + (barged: boolean, stoppedDuringSetup = false) => { if (barged || !awaitingSpokenResponseRef.current) { awaitingSpokenResponseRef.current = false consumePendingResponse() @@ -207,7 +282,16 @@ export function useVoiceConversation({ dropSpeechSession() - if (enabledRef.current) { + // If stopVoicePlayback() was called externally (Stop button, end), the + // voice-playback sequence has advanced past what we captured at speech + // start — don't auto-start the next sentence, the user chose to stop. + const stoppedByUser = + stoppedDuringSetup || + (speechStartSequenceRef.current > 0 && $voicePlayback.get().sequence > speechStartSequenceRef.current) + + speechStartSequenceRef.current = 0 + + if (enabledRef.current && !stoppedByUser) { pendingStartRef.current = true } @@ -248,6 +332,25 @@ export function useVoiceConversation({ return } + // A spoken stop command while barging means "stop everything" — the + // turn/playback was already cut at trip time; now end the conversation + // instead of submitting "stop" as a new prompt. + if (isVoiceStopCommand(transcript)) { + dropSpeechSession() + setStatus('idle') + onStopWordRef.current?.() + + return + } + + // A generation-phase barge interrupted the in-flight turn; the submit + // path refuses while `busy`, so wait for the interrupt to settle. + const deadline = Date.now() + INTERRUPT_SETTLE_TIMEOUT_MS + + while (busyRef.current && Date.now() < deadline) { + await new Promise(resolve => window.setTimeout(resolve, 100)) + } + awaitingSpokenResponseRef.current = true dropSpeechSession() consumePendingResponse() @@ -261,24 +364,46 @@ export function useVoiceConversation({ [consumePendingResponse, onSubmit, onTranscribeAudio, voiceCopy.transcriptionFailed] ) - /** Barge-in monitor wiring shared by the live and fallback speech paths. */ - const openBargeMonitor = useCallback( - (onBarge: () => void) => - monitorSpeechDuringPlayback({ - onSpeech: () => { - bargeCapturePendingRef.current = true - onBarge() - markVoicePlaybackInterrupted() - stopVoicePlayback() - }, - onUtterance: audio => { - bargeCapturePendingRef.current = false - stopBargeMonitorRef.current = null - void submitCapturedUtterance(audio) + /** + * Full-duplex barge-in monitor for the WHOLE agent turn: armed at submit, + * live through generation (thinking) AND playback (speaking). + * + * - generation phase (`busy`): speech interrupts the in-flight turn via + * `onInterrupt` — the same seam as the Stop button — and cuts any TTS that + * managed to start, so the stale reply never speaks. + * - playback phase: speech cuts playback and the captured interruption is + * transcribed and submitted as the next turn. + * + * Idempotent — one monitor owns the mic per turn; re-arming while one is + * live is a no-op (the live/fallback speech paths and the turn-drive effect + * all call this). + */ + const ensureBargeMonitor = useCallback(() => { + if (stopBargeMonitorRef.current) { + return + } + + stopBargeMonitorRef.current = monitorSpeechDuringPlayback({ + isPlaying: () => $voicePlayback.get().status === 'speaking', + onSpeech: () => { + bargeCapturePendingRef.current = true + bargedRef.current = true + markVoicePlaybackInterrupted() + stopVoicePlayback() + + if (busyRef.current) { + // Mid-generation: stop the in-flight turn so the captured utterance + // becomes the next one instead of queueing behind a stale reply. + void onInterruptRef.current?.() } - }), - [submitCapturedUtterance] - ) + }, + onUtterance: audio => { + bargeCapturePendingRef.current = false + stopBargeMonitorRef.current = null + void submitCapturedUtterance(audio) + } + }) + }, [submitCapturedUtterance]) /** Push any new reply text into the live session; finish when complete. */ const feedSpeechSession = useCallback( @@ -330,26 +455,29 @@ export function useVoiceConversation({ return } - let barged = false + // The full-duplex monitor is normally already live (armed at submit); + // this is a safety net for read-aloud-style entries into the loop. + ensureBargeMonitor() - stopBargeMonitorRef.current?.() - stopBargeMonitorRef.current = openBargeMonitor(() => { - barged = true - }) + const playback = playSpeechText(response.text, { source: 'voice-conversation' }) + // playSpeechText performs its normal cleanup synchronously before + // returning. Capture the sequence after that internal increment so + // only a later, external stop suppresses the next listen cycle. + speechStartSequenceRef.current = $voicePlayback.get().sequence - void playSpeechText(response.text, { source: 'voice-conversation' }) + void playback .catch(error => notifyError(error, voiceCopy.playbackFailed)) .finally(() => { if (responseIdRef.current === responseId) { awaitingSpokenResponseRef.current = false - settleAfterSpeech(barged) + settleAfterSpeech(bargedRef.current) } }) } poll() }, - [openBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed] + [ensureBargeMonitor, pendingResponse, settleAfterSpeech, voiceCopy.playbackFailed] ) /** @@ -359,19 +487,17 @@ export function useVoiceConversation({ */ const openLiveSpeech = useCallback( (responseId: string) => { + const sequenceBeforeStart = $voicePlayback.get().sequence + responseIdRef.current = responseId spokenSourceLengthRef.current = 0 setStatus('speaking') - let barged = false - // VAD barge-in: the user talking over the reply cuts playback, drops // the not-yet-spoken remainder, AND keeps capturing — the interruption // is transcribed from its first syllable instead of losing the opening - // words to a mic re-open. - stopBargeMonitorRef.current = openBargeMonitor(() => { - barged = true - }) + // words to a mic re-open. Usually already live (armed at submit). + ensureBargeMonitor() void (async () => { const session = await startSpeechStream({ source: 'voice-conversation' }) @@ -386,6 +512,16 @@ export function useVoiceConversation({ } if (!session) { + // Stream discovery can also fail after an explicit Stop landed + // during its async URL lookup. In that case, do not turn the stopped + // live attempt into fresh fallback playback. + if ($voicePlayback.get().sequence > sequenceBeforeStart) { + awaitingSpokenResponseRef.current = false + settleAfterSpeech(false, true) + + return + } + // No streaming backend/provider: speak the whole reply once it lands. speechSessionRef.current = null awaitFallbackSpeech(responseId) @@ -393,8 +529,24 @@ export function useVoiceConversation({ return } + // startSpeechStream calls stopVoicePlayback once after its async URL + // lookup. A second sequence bump means the user pressed Stop while + // setup was still pending. Do not absorb that explicit stop into the + // post-start baseline or allow the new session to play. + const sequenceAfterStart = $voicePlayback.get().sequence + const stoppedDuringStart = sequenceAfterStart > sequenceBeforeStart + 1 + + speechStartSequenceRef.current = sequenceAfterStart speechSessionRef.current = session + if (stoppedDuringStart) { + stopVoicePlayback() + awaitingSpokenResponseRef.current = false + settleAfterSpeech(false, true) + + return + } + // Timer-driven feed: reply text flows into the session at delta rate // regardless of React render cadence. const feedTimer = window.setInterval(() => feedSpeechSession(responseId), 150) @@ -414,10 +566,10 @@ export function useVoiceConversation({ } awaitingSpokenResponseRef.current = false - settleAfterSpeech(barged) + settleAfterSpeech(bargedRef.current) })() }, - [awaitFallbackSpeech, feedSpeechSession, openBargeMonitor, settleAfterSpeech] + [awaitFallbackSpeech, ensureBargeMonitor, feedSpeechSession, settleAfterSpeech] ) const start = useCallback(async () => { @@ -505,15 +657,39 @@ export function useVoiceConversation({ return () => window.removeEventListener('keydown', onKeyDown, { capture: true }) }, [enabled, stopTurn]) + // Ambient "thinking" sound: while the agent works (status 'thinking') no + // audio flows, which reads as dead air mid-conversation. Calm bubble blips + // fill the gap; they stop the INSTANT speech starts, the mic re-arms, or the + // conversation ends. Gated by voice.thinking_sound + the shared sound mute. + useEffect(() => { + if (enabled && !muted && status === 'thinking') { + startThinkingSound() + + return stopThinkingSound + } + + stopThinkingSound() + + return undefined + }, [enabled, muted, status]) + // Drive the loop: when a voice-submitted reply appears, open a live speech // session (which feeds itself from then on). Otherwise start listening when // idle between turns. + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!enabled || muted) { return } if (awaitingSpokenResponseRef.current && status !== 'speaking') { + // Generation phase: the turn is in flight but no reply audio exists + // yet. Keep the mic live so speech can interrupt the model mid- + // generation (full-duplex) instead of going deaf until playback. + if (status === 'thinking' && (busy || bargeCapturePendingRef.current)) { + ensureBargeMonitor() + } + const response = pendingResponse() if (response) { @@ -522,8 +698,9 @@ export function useVoiceConversation({ return } - if (!busy && status === 'thinking') { - // Turn finished without any speakable reply (tool-only, error). + if (!busy && status === 'thinking' && !bargeCapturePendingRef.current) { + // Turn finished without any speakable reply (tool-only, error). A + // live barge capture owns the loop instead — it submits or resumes. awaitingSpokenResponseRef.current = false dropSpeechSession() pendingStartRef.current = true @@ -540,8 +717,9 @@ export function useVoiceConversation({ if (pendingStartRef.current) { void startListening() } - }, [busy, enabled, muted, openLiveSpeech, pendingResponse, startListening, status]) + }, [busy, enabled, muted, ensureBargeMonitor, openLiveSpeech, pendingResponse, startListening, status]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (enabled && !wasEnabledRef.current) { void start() diff --git a/apps/desktop/src/app/chat/composer/index.tsx b/apps/desktop/src/app/chat/composer/index.tsx index 1ce110f4e098..28dbcf23a67b 100644 --- a/apps/desktop/src/app/chat/composer/index.tsx +++ b/apps/desktop/src/app/chat/composer/index.tsx @@ -1,8 +1,8 @@ import { ComposerPrimitive } from '@assistant-ui/react' import { useStore } from '@nanostores/react' -import { type ClipboardEvent, type FormEvent, type KeyboardEvent, useCallback, useEffect, useRef } from 'react' +import { type ClipboardEvent, type FormEvent, type KeyboardEvent, useCallback, useEffect, useMemo, useRef } from 'react' -import { composerFill, composerSurfaceGlass } from '@/components/chat/composer-dock' +import { composerFill, composerFloatingStrip, composerSurfaceGlass } from '@/components/chat/composer-dock' import { Button } from '@/components/ui/button' import { Slot as ContribSlot } from '@/contrib/react/slot' import { useI18n } from '@/i18n' @@ -11,7 +11,8 @@ import { sanitizeComposerInput } from '@/lib/composer-input-sanitize' import { DATA_IMAGE_URL_RE } from '@/lib/embedded-images' import { triggerHaptic } from '@/lib/haptics' import { cn } from '@/lib/utils' -import { $compactionActive } from '@/store/compaction' +import { interceptsTypedVoiceStop } from '@/lib/voice-stop-word' +import { sessionCompacting } from '@/store/compaction' import { browseBackward, browseForward, deriveUserHistory, isBrowsingHistory } from '@/store/composer-input-history' import { POPOUT_WIDTH_REM } from '@/store/composer-popout' import { parkQueuedPrompts, removeQueuedPrompt, unparkQueuedPrompts } from '@/store/composer-queue' @@ -22,10 +23,16 @@ import { $autoSpeakReplies } from '@/store/voice-prefs' import { useTheme } from '@/themes' import { AttachmentList } from './attachments' -import { COMPOSER_FADE_BACKGROUND, type QueueEditState, slashArgStage } from './composer-utils' +import { + acceptsTriggerCompletion, + COMPOSER_FADE_BACKGROUND, + type QueueEditState, + slashArgStage +} from './composer-utils' import { ContextMenu } from './context-menu' import { COMPOSER_AREAS, runComposerMiddleware } from './contrib' import { ComposerControls } from './controls' +import { ComposerDirectiveActions } from './directive-actions' import { COMPOSER_DROP_ACTIVE_CLASS, COMPOSER_DROP_FADE_CLASS } from './drop-affordance' import { markActiveComposer } from './focus' import { HelpHint } from './help-hint' @@ -40,26 +47,34 @@ import { useComposerPopout } from './hooks/use-composer-popout' import { useComposerQueue } from './hooks/use-composer-queue' import { useComposerSubmit } from './hooks/use-composer-submit' import { useComposerTrigger } from './hooks/use-composer-trigger' +import { useComposerUndo } from './hooks/use-composer-undo' import { useComposerUrlDialog } from './hooks/use-composer-url-dialog' import { useComposerVoice } from './hooks/use-composer-voice' +import { useEmojiCompletions } from './hooks/use-emoji-completions' +import { useComposerMicroActions } from './hooks/use-micro-actions' import { useSlashCompletions } from './hooks/use-slash-completions' import { useSessionStatusPresence } from './hooks/use-status-presence' +import { ActionBadges } from './micro-actions' +import { chipTypedPathOnSpace, pathifyRefs } from './path-refs' import { QueuePanel } from './queue-panel' import { + beginComposerComposition, composerPlainText, deleteChipBeforeCaret, deleteSelectionInEditor, - insertPlainTextAtCaret, + insertComposerContentsAtCaret, normalizeComposerEditorDom, RICH_INPUT_SLOT } from './rich-editor' import { useComposerScope } from './scope' import { ComposerStatusStack } from './status-stack' import { CodingStatusRow } from './status-stack/coding-row' -import { extractClipboardImageBlobs } from './text-utils' +import { extractClipboardImageBlobs, openDirectiveScope } from './text-utils' import { ComposerTriggerPopover } from './trigger-popover' import type { ChatBarProps } from './types' +import { isRedoShortcut, isUndoShortcut } from './undo-history' import { UrlDialog } from './url-dialog' +import { chipTypedUrlOnSpace, linkifyUrls } from './url-refs' import { VoiceActivity, VoicePlaybackActivity } from './voice-activity' export function ChatBar({ @@ -85,11 +100,32 @@ export function ChatBar({ onSubmit: onSubmitProp, onTranscribeAudio }: ChatBarProps) { + // Typed stop phrase during an active voice conversation ends it — same + // semantics as SAYING "stop" (voice-stop-word.ts) or clicking the pill's + // end control. Populated after useComposerVoice below (the submit wrapper + // is created first); render-time assignment keeps the ref current. + const voiceStopRef = useRef<{ active: boolean; end: () => void }>({ active: false, end: () => {} }) + // Every send (typed, queued, voice) passes through the contributed // middleware chain first — rewrite / pass-through / cancel. Empty chain = // exact pass-through, so surfaces without contributions are byte-identical. const onSubmit = useCallback( async (value, options) => { + // Bare stop phrase typed while the voice conversation is live: end the + // conversation (mic off, pill dismissed) instead of sending "stop" to + // the agent. Spoken transcripts are already stop-checked inside + // use-voice-conversation, so this only catches typed/queued sends. + // Outside a voice conversation, typed "stop" is a normal message. + const voiceStop = voiceStopRef.current + + if (interceptsTypedVoiceStop(voiceStop.active, value, options?.attachments?.length ?? 0)) { + voiceStop.end() + + // Consumed (not rejected): report accepted so the submit engine + // clears the draft instead of restoring "stop" into the composer. + return true + } + const draft = await runComposerMiddleware({ text: value, attachments: options?.attachments }) if (!draft) { @@ -105,7 +141,7 @@ export function ChatBar({ // focus-bus key, and awaiting-input edge. Main scope = the legacy globals. const scope = useComposerScope() const attachments = useStore(scope.attachments.$attachments) - const compacting = useStore($compactionActive) + const compacting = useStore(useMemo(() => sessionCompacting(sessionId ?? null), [sessionId])) const scrolledUp = useStore($threadScrolledUp) const autoSpeak = useStore($autoSpeakReplies) // The turn is parked on the user (clarify / approval / sudo / secret). Esc must @@ -124,7 +160,14 @@ export function ChatBar({ // every per-item status mutation or other sessions' churn (see the hook). const statusPresent = useSessionStatusPresence(statusSessionId) + // Publishes contributed micro actions for this session; the status stack + // renders them as the pill strip at the top of the overlay lane. + useComposerMicroActions(statusSessionId, busy) + const composerRef = useRef(null) + // The dock wraps the strips + status stack + composer; the thread's bottom + // clearance measures this, while the pop-out drag still tracks the composer. + const composerDockRef = useRef(null) const composerSurfaceRef = useRef(null) // Pop-out engine: docked↔floating state, dock/float/toggle, drag gestures, and @@ -148,6 +191,7 @@ export function ChatBar({ const { availableThemes, themeName } = useTheme() const at = useAtCompletions({ gateway: gateway ?? null, sessionId: sessionId ?? null, cwd: cwd ?? null }) const slash = useSlashCompletions({ activeSkin: themeName, gateway: gateway ?? null, skinThemes: availableThemes }) + const emoji = useEmojiCompletions() const { t } = useI18n() const gatewayState = useStore($gatewayState) @@ -172,9 +216,24 @@ export function ChatBar({ requestMainFocus, sessionIdRef, setComposerText, - stashAt + stashAt, + syncDraftFromEditor } = useComposerDraft({ activeQueueSessionKey, focusKey, inputDisabled, queueEditRef, sessionId }) + // Undo/redo. The rich editor bypasses Chromium's editing pipeline for speed, + // which also bypasses its undo stack — so we own the stack and every edit + // path below banks its pre-edit state through `recordUndoPoint`. + const { recordUndoPoint, redo, resetUndoHistory, undo, withUndoPoint } = useComposerUndo({ + editorRef, + syncDraftFromEditor + }) + + // Prior history belongs to the draft that just left — undoing into another + // conversation's text is worse than having none. + useEffect(() => { + resetUndoHistory() + }, [activeQueueSessionKey, resetUndoHistory]) + // "Add URL" dialog — open/value state, autofocus, and submit (host onAddUrl or // an @url: directive into the draft). const { openUrlDialog, setUrlOpen, setUrlValue, submitUrl, urlInputRef, urlOpen, urlValue } = useComposerUrlDialog({ @@ -226,7 +285,14 @@ export function ChatBar({ return onCancel() }, [activeQueueSessionKeyRef, onCancel]) - const { compactPill, stacked } = useComposerMetrics({ composerRef, composerSurfaceRef, editorRef, poppedOut }) + const { compactPill, stacked } = useComposerMetrics({ + composerDockRef, + composerRef, + composerSurfaceRef, + editorRef, + poppedOut + }) + const hasComposerPayload = hasText || attachments.length > 0 const canSubmit = busy || hasComposerPayload @@ -281,17 +347,21 @@ export function ChatBar({ // this API; keyup uses triggerKeyConsumedRef to skip its refresh. const { argStageEmpty, + ascendTriggerPath, closeTrigger, commitTypedSlashDirective, + moveTriggerActive, refreshTrigger, replaceTriggerWithChip, setTriggerActive, + slashFreeTextArgStage, trigger, triggerActive, + triggerActiveExplicit, triggerItems, triggerKeyConsumedRef, triggerLoading - } = useComposerTrigger({ at, draftRef, editorRef, requestMainFocus, setComposerText, slash }) + } = useComposerTrigger({ at, draftRef, editorRef, emoji, recordUndoPoint, requestMainFocus, setComposerText, slash }) // Pull the live contentEditable text into draftRef + the AUI composer state // (which drives `hasComposerPayload` → the send button). Shared by the input @@ -356,21 +426,32 @@ export function ChatBar({ scheduleFlushEditorToDraft(event.currentTarget) } + // Native typing/deleting mutates the DOM through Chromium's editing pipeline, + // whose undo stack we've taken over — so bank the pre-edit state here, before + // the change lands. `beforeinput` is the only hook that still sees the old + // text. Consecutive keystrokes coalesce into one entry, so ⌘Z steps back by a + // burst rather than a character. + const handleEditorBeforeInput = (event: FormEvent) => { + const inputType = (event.nativeEvent as InputEvent).inputType + + // Undo/redo are ours (handled in useComposerUndo + keydown), and IME preedit + // is not a committed edit — compositionend is where that text becomes real. + if (inputType === 'historyUndo' || inputType === 'historyRedo' || composingRef.current) { + return + } + + recordUndoPoint({ coalesce: inputType === 'insertText' || inputType === 'deleteContentBackward' }) + } + const handlePaste = (event: ClipboardEvent) => { const imageBlobs = extractClipboardImageBlobs(event.clipboardData) - if (imageBlobs.length > 0) { - event.preventDefault() - - if (onAttachImageBlob) { - triggerHaptic('selection') + if (imageBlobs.length > 0 && onAttachImageBlob) { + triggerHaptic('selection') - for (const blob of imageBlobs) { - void onAttachImageBlob(blob) - } + for (const blob of imageBlobs) { + void onAttachImageBlob(blob) } - - return } // Trim surrounding whitespace so a copy that dragged along leading/trailing @@ -382,6 +463,10 @@ export function ChatBar({ if (!pastedText) { event.preventDefault() + if (imageBlobs.length > 0) { + return + } + // Under WSL2/WSLg the Windows host clipboard doesn't bridge *images* to // the Linux clipboard the DOM paste event reads, so a host screenshot // arrives as an empty paste (no blobs, no text). Fall back to the main @@ -402,7 +487,17 @@ export function ChatBar({ } event.preventDefault() - insertPlainTextAtCaret(event.currentTarget, pastedText) + + // Links in the paste land as `@url:` chips rather than a wall of URL text — + // the same reference the "Add URL" dialog inserts, parsed in place so a link + // mid-sentence keeps its position. Bare `@path` tokens promote the same way. + // A paste into an open `@url:`/`@file:` scope CONSUMES that scope instead of + // stacking on it — the scope is the browse mode the user is pasting into, + // not text they typed and want to keep (`@url:@url:\`https://…\``). + const scope = openDirectiveScope(event.currentTarget) + + recordUndoPoint() + insertComposerContentsAtCaret(event.currentTarget, pathifyRefs(linkifyUrls(pastedText)), scope) scheduleFlushEditorToDraft(event.currentTarget) } @@ -416,6 +511,23 @@ export function ChatBar({ return } + // Undo/redo before anything else — we own the stack (see useComposerUndo), + // so these never reach Chromium's native history, which has no record of + // the Range-based edits the rich editor makes. + if (isUndoShortcut(event.nativeEvent)) { + event.preventDefault() + undo() + + return + } + + if (isRedoShortcut(event.nativeEvent)) { + event.preventDefault() + redo() + + return + } + // Plain Backspace right after a directive chip: remove the chip + its // auto-inserted trailing space as one unit, so deleting a directive never // leaves an orphaned space. (Modified backspaces stay native.) @@ -424,7 +536,7 @@ export function ChatBar({ !event.metaKey && !event.ctrlKey && !event.altKey && - deleteChipBeforeCaret(event.currentTarget) + withUndoPoint(() => deleteChipBeforeCaret(event.currentTarget)) ) { event.preventDefault() flushEditorToDraft(event.currentTarget) @@ -434,7 +546,29 @@ export function ChatBar({ // Non-collapsed Backspace/Delete: native selection-delete is ~O(n²) on large // drafts (Ctrl+A → Delete froze ~1.3s). Collapsed carets fall through. - if ((event.key === 'Backspace' || event.key === 'Delete') && deleteSelectionInEditor(event.currentTarget)) { + if ( + (event.key === 'Backspace' || event.key === 'Delete') && + withUndoPoint(() => deleteSelectionInEditor(event.currentTarget)) + ) { + event.preventDefault() + flushEditorToDraft(event.currentTarget) + + return + } + + // A typed link finished with a space chips like a pasted one — the space + // itself rides along inside the insert. + if (withUndoPoint(() => chipTypedUrlOnSpace(event))) { + event.preventDefault() + flushEditorToDraft(event.currentTarget) + + return + } + + // Same for a bare `@path` — a hand-typed or Tab-descended path chips into + // the `@file:`/`@folder:` ref it means, instead of submitting as plain text + // the backend never resolves. + if (withUndoPoint(() => chipTypedPathOnSpace(event))) { event.preventDefault() flushEditorToDraft(event.currentTarget) @@ -453,11 +587,22 @@ export function ChatBar({ return } + // The popover is open but its items are still in flight (debounce + RPC). + // Tab must not fall through to the browser — it would move focus out of + // the composer mid-completion, which reads as the popover "eating" the + // keypress. Swallow it; the refresh lands with the items. + if (trigger && triggerLoading && triggerItems.length === 0 && event.key === 'Tab') { + event.preventDefault() + triggerKeyConsumedRef.current = true + + return + } + if (trigger && triggerItems.length > 0) { if (event.key === 'ArrowDown') { event.preventDefault() triggerKeyConsumedRef.current = true - setTriggerActive(idx => (idx + 1) % triggerItems.length) + moveTriggerActive(1) return } @@ -465,18 +610,21 @@ export function ChatBar({ if (event.key === 'ArrowUp') { event.preventDefault() triggerKeyConsumedRef.current = true - setTriggerActive(idx => (idx - 1 + triggerItems.length) % triggerItems.length) + moveTriggerActive(-1) return } - // Enter / Tab / Space all accept the highlighted item: a no-arg command - // commits its directive chip, an arg-taking command expands to its - // options step, and an arg option commits the full `/cmd arg` chip. Space - // is slash-only (an `@` mention takes a literal space) and gated to a - // non-empty query so a bare `/ ` still types a space. - const acceptOnSpace = event.key === ' ' && trigger.kind === '/' && Boolean(trigger.query.trim()) - const accept = event.key === 'Enter' || event.key === 'Tab' || acceptOnSpace + // Accepting the highlighted item: a no-arg command commits its directive + // chip, an arg-taking command expands to its options step, and an arg + // option commits the full `/cmd arg` chip. + const accept = acceptsTriggerCompletion({ + activeExplicit: triggerActiveExplicit, + freeTextArgStage: slashFreeTextArgStage, + key: event.key, + kind: trigger.kind, + query: trigger.query + }) if (accept) { event.preventDefault() @@ -484,12 +632,24 @@ export function ChatBar({ const item = triggerItems[triggerActive] if (item) { - replaceTriggerWithChip(item) + // Tab means "go deeper" on a folder; Enter means "I want this one". + // Everything else treats them alike. + replaceTriggerWithChip(item, { descend: event.key === 'Tab' }) } return } + // Backspace climbs out of an `@` path one segment at a time, mirroring + // Tab's one-key descent. Only when the caret sits at the end of the + // token — mid-token editing keeps normal character deletion. + if (event.key === 'Backspace' && !event.metaKey && !event.altKey && ascendTriggerPath()) { + event.preventDefault() + triggerKeyConsumedRef.current = true + + return + } + if (event.key === 'Escape') { event.preventDefault() triggerKeyConsumedRef.current = true @@ -510,11 +670,12 @@ export function ChatBar({ slashArgStage(trigger.query) && trigger.query.trim() ) { - event.preventDefault() - triggerKeyConsumedRef.current = true - commitTypedSlashDirective() + if (commitTypedSlashDirective()) { + event.preventDefault() + triggerKeyConsumedRef.current = true - return + return + } } // ArrowUp/ArrowDown navigate, in priority order: the queue (edit entries in @@ -551,7 +712,7 @@ export function ChatBar({ // $messages is read imperatively (not subscribed) so the composer // doesn't re-render on every streaming delta flush. - const history = deriveUserHistory(scope.readMessages(), chatMessageText) + const history = deriveUserHistory(scope.$messages.get(), chatMessageText) const entry = browseBackward(sessionId, currentDraft, history) if (entry !== null) { @@ -576,7 +737,7 @@ export function ChatBar({ event.preventDefault() triggerKeyConsumedRef.current = true - const history = deriveUserHistory(scope.readMessages(), chatMessageText) + const history = deriveUserHistory(scope.$messages.get(), chatMessageText) const result = browseForward(sessionId, history) if (result !== null) { @@ -630,12 +791,21 @@ export function ChatBar({ return } - // Empty Enter while busy is a no-op — interrupting is explicit (Stop/Esc), - // never a stray Enter after sending. With a payload, submitDraft queues it. - // Gate on the live DOM payload (not the render-lagged composer state) so a - // message typed fast / via IME while busy still reaches submitDraft() and - // gets queued instead of being mistaken for an empty Enter. + // Empty Enter while busy. With prompts queued this is the double-send: + // the first Enter put the words in the queue, a second sends them now + // (promote + interrupt + drain on settle), mirroring the idle empty-Enter + // drain above. With nothing queued it stays a no-op — interrupting is + // explicit (Stop/Esc), never a stray Enter after sending. Gate on the live + // DOM payload (not the render-lagged composer state) so a message typed + // fast / via IME while busy still reaches submitDraft() and gets queued + // instead of being mistaken for an empty Enter. if (busy && !hasLivePayload) { + const head = queuedPrompts.find(entry => entry.id !== queueEdit?.entryId) + + if (head) { + sendQueuedNow(head.id) + } + return } @@ -715,12 +885,19 @@ export function ChatBar({ focusInput, insertText, maxRecordingSeconds, + // Voice barge-in mid-generation halts the run like the Stop button. + onInterrupt: haltRun, onSubmit, onTranscribeAudio, sessionId, target: scope.target }) + // Keep the typed-stop interceptor (see onSubmit above) in sync with the + // live conversation state. Render-time ref assignment, same pattern as + // dispatchSubmitRef — no effect needed for a plain mirror. + voiceStopRef.current = { active: voiceConversationActive, end: endConversation } + const contextMenu = ( window.setTimeout(closeTrigger, 80)} onCompositionEnd={event => { composingRef.current = false @@ -790,8 +967,13 @@ export function ChatBar({ // until an unrelated edit forces a sync (#39614). flushEditorToDraft(event.currentTarget) }} - onCompositionStart={() => { + onCompositionStart={event => { composingRef.current = true + + // Input events are skipped for the rest of the composition, so + // nothing else would clear the empty marker until it ends — and the + // hint would sit behind the preedit text the whole time (#75960). + beginComposerComposition(event.currentTarget) }} onDragOver={handleInputDragOver} onDrop={handleInputDrop} @@ -806,6 +988,7 @@ export function ChatBar({ spellCheck={false} suppressContentEditableWarning /> + {/* assistant-ui requires ComposerPrimitive.Input somewhere in the tree so the composer-state binding (text + IME + paste + form-submit hookup) wires up. We render the real input UI ourselves above via the @@ -858,36 +1041,27 @@ export function ChatBar({ /> )} - { - e.preventDefault() - - if (composingRef.current) { - return - } - - submitDraft() - }} - ref={composerRef} + // Measured for the thread's bottom clearance: the dock is the box + // that contains the strips, the status stack, AND the composer, so + // one measurement covers everything the thread must clear. + ref={composerDockRef} style={ poppedOut ? { @@ -899,22 +1073,16 @@ export function ChatBar({ : undefined } > - {isHelpHint && } - {trigger && !argStageEmpty && ( - - )} + {/* Aligned to the composer SURFACE, which sits inside the composer's + 5px transparent grab margin — so both strips carry the same inset + and share one left edge with it. */} +
+ +
{/* Session-scoped status stack (todos, subagents, background tasks, - queue). Out of flow so it never inflates the composer's measured - height; it overlays the chat instead of pushing it, and publishes - its own --status-stack-measured-height so the thread's clearance - accounts for it. Collapses to nothing when every status is empty. */} + queue). An in-flow dock child: the dock is bottom-anchored, so it + grows upward over the thread and the dock's own measurement covers + it. Collapses to nothing when every status is empty. */} 0 ? ( @@ -944,116 +1112,166 @@ export function ChatBar({ } sessionId={statusSessionId} /> - {!poppedOut && ( -
- )} - {/* Drag region: covers the transparent grab margin around the surface. + { + e.preventDefault() + + if (composingRef.current) { + return + } + + submitDraft() + }} + ref={composerRef} + > + {isHelpHint && } + {trigger && !argStageEmpty && ( + + )} + {!poppedOut && ( +
+ )} + {/* Drag region: covers the transparent grab margin around the surface. The surface sits on top (z-4) so only the exposed ring receives this element's hover/cursor — grab cursor + a diagonal hatch (/////) appear when you hover the draggable margin, never over the input. The hatch pattern + opacity ladder live in styles.css. */} - {popoutAllowed && ( -
- )} -
-
+ {popoutAllowed && (
- + )} +
- {/* Contribution seams: banners above, a row below, inline - additions beside the "+" menu and before the controls. - All four render nothing until something contributes. */} - - - - {queueEdit && editingQueuedPrompt && ( -
-
- {t.composer.editingQueuedInComposer} -
-
- - -
-
- )} - {attachments.length > 0 && }
+ toggleReview(scope.target === 'main' ? null : (cwd ?? null))} + onOpenWorktree={openInWorktree} + onSwitchBranch={handleSwitchBranch} + repoPath={cwd} + /> +
-
- {contextMenu} - -
-
{input}
-
- - {controls} + {/* Contribution seams: banners above, a row below, inline + additions beside the "+" menu and before the controls. + All four render nothing until something contributes. */} + + + + {queueEdit && editingQueuedPrompt && ( +
+
+ {t.composer.editingQueuedInComposer} +
+
+ + +
+
+ )} + {attachments.length > 0 && } +
+
+ {contextMenu} + +
+
{input}
+
+ + {controls} +
+
-
+ + {/* Underside: chrome-free strip BELOW the composer. Outside the root + for the same reason as the micro actions — it must not fall inside + the pop-out drag region. Same px as the strip above, so the two + bracket the composer on one vertical line. */} +
+
- +
{ + it('marks any element as a reference of a given kind', () => { + expect(refAttrs('file')).toEqual({ className: 'ref', 'data-ref': 'file' }) + expect(refAttrsHtml('skill')).toBe('class="ref" data-ref="skill"') + }) + + it('an unkinded reference is a plain link, not a broken one', () => { + // A bare external link has no kind — it keeps the default link colour + // rather than being tagged with a wrong one. + expect(refAttrs()).toEqual({ className: 'ref' }) + expect(refAttrsHtml()).toBe('class="ref"') + }) + + it('normalises an unknown kind instead of emitting it raw', () => { + // A kind CSS has no rule for would silently render unstyled; coercing to + // `other` keeps it inside the system. + expect(refAttrs('wat')['data-ref']).toBe('other') + expect(referenceKind('wat')).toBe('other') + }) + + it('ships no colour from TypeScript — the theme owns every accent', () => { + // The whole point of keying on `data-ref`: a skin restyles all references + // at once, and no hex or color-mix() is hardcoded in a component. + for (const [kind, style] of Object.entries(REFERENCE_STYLES)) { + expect(style, `${kind} must not carry a colour`).not.toHaveProperty('color') + } + + expect(JSON.stringify(refAttrs('url'))).not.toMatch(/color|#[0-9a-f]{3}/i) + }) + + it('gives every kind a glyph and a label', () => { + for (const [kind, style] of Object.entries(REFERENCE_STYLES)) { + expect(style.codicon, `${kind} codicon`).toBeTruthy() + expect(style.label, `${kind} label`).toBeTruthy() + + // Emoji rows render the emoji itself instead of a glyph. + if (kind !== 'emoji') { + expect(style.paths.length, `${kind} paths`).toBeGreaterThan(0) + } + } + }) + + it('keeps commands and skills visually distinct', () => { + // Different data-ref values, so the stylesheet can accent them apart. + expect(refAttrs('skill')['data-ref']).not.toBe(refAttrs('command')['data-ref']) + expect(referenceStyle('skill').codicon).not.toBe(referenceStyle('command').codicon) + }) +}) + +describe('references are text, not badges', () => { + it('carries no layout, padding, or background of its own', () => { + // Everything visual lives in the stylesheet. If a component starts adding + // its own chrome here, that's the drift this system exists to prevent. + const { className } = refAttrs('file') + + expect(className).toBe('ref') + + for (const chrome of ['bg-', 'rounded', 'px-', 'py-', 'border', 'inline-flex', 'text-[']) { + expect(className).not.toContain(chrome) + } + }) +}) diff --git a/apps/desktop/src/app/chat/composer/inline-refs.ts b/apps/desktop/src/app/chat/composer/inline-refs.ts index 5fd62f4cc94a..5e282f6a040a 100644 --- a/apps/desktop/src/app/chat/composer/inline-refs.ts +++ b/apps/desktop/src/app/chat/composer/inline-refs.ts @@ -4,7 +4,13 @@ import { contextPath } from '@/lib/chat-runtime' import type { DroppedFile } from '../hooks/use-composer-actions' -import { composerPlainText, normalizeComposerEditorDom, placeCaretEnd, refChipElement } from './rich-editor' +import { + composerPlainText, + normalizeComposerEditorDom, + placeCaretEnd, + refChipElement, + RICH_INPUT_SLOT +} from './rich-editor' /** A chip to insert: a raw `@kind:value` string, or a typed value + display label. */ export type InlineRefInput = string | { kind: string; label?: string; value: string } @@ -92,7 +98,12 @@ function plainTextInRange(editor: HTMLDivElement, range: Range, edge: 'after' | slice.setStart(range.endContainer, range.endOffset) } + // Carry the editor's slot marker: composerPlainText appends a trailing "\n" + // to any other block element, so a bare
made `beforeText` always look + // like it ended in whitespace and the separating space was never inserted — + // a chip dropped after a word came out glued to it (`review@file:...`). const container = document.createElement('div') + container.dataset.slot = RICH_INPUT_SLOT container.appendChild(slice.cloneContents()) return composerPlainText(container) diff --git a/apps/desktop/src/app/chat/composer/micro-actions.tsx b/apps/desktop/src/app/chat/composer/micro-actions.tsx new file mode 100644 index 000000000000..65e3729eadad --- /dev/null +++ b/apps/desktop/src/app/chat/composer/micro-actions.tsx @@ -0,0 +1,71 @@ +import { memo, useState } from 'react' + +import { composerFloatingPill } from '@/components/chat/composer-dock' +import { Codicon } from '@/components/ui/codicon' +import { useSessionSlice } from '@/lib/use-session-slice' +import { cn } from '@/lib/utils' +import { $composerActionsBySession, type ComposerAction } from '@/store/composer-actions' +import { notifyError } from '@/store/notifications' + +/** + * Floating pill — the shared treatment for a control that sits over the + * composer (`composerFloatingPill`), plus this strip's own width cap and + * disabled state. + * + * NEVER `pointer-events-none`, not even when disabled. The pop-out drag region + * is an `absolute` sibling behind these pills, so a pill that stops taking + * pointer events hands the hit test straight to it — and a dead-looking badge + * becomes a grab handle that floats the composer. + */ +const PILL = cn( + composerFloatingPill, + 'max-w-56', + 'disabled:cursor-default disabled:opacity-50 disabled:hover:bg-(--composer-fill)', + 'focus-visible:outline-none focus-visible:ring-[0.1875rem] focus-visible:ring-ring/50' +) + +/** + * The micro-action pills. Layout-free on purpose — the composer owns the strip + * (`composerFloatingStrip`), this owns only the pills, so the strip above the + * surface and the `composer.underside` strip below it can't drift apart. + */ +export const ActionBadges = memo(function ActionBadges({ sessionId }: { sessionId: null | string }) { + const actions = useSessionSlice($composerActionsBySession, sessionId) + // A pill can kick off async work (a gateway call, a submit). Track which one + // is in flight so it can spin and lock instead of double-firing. + const [runningId, setRunningId] = useState(null) + + const run = async (action: ComposerAction) => { + if (runningId || !sessionId) { + return + } + + setRunningId(action.id) + + try { + await action.run(sessionId) + } catch (error) { + notifyError(error, action.label) + } finally { + setRunningId(null) + } + } + + return actions.map(action => { + const running = runningId === action.id + const glyph = running ? 'loading' : action.icon + + return ( + + ) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/model-pill.tsx b/apps/desktop/src/app/chat/composer/model-pill.tsx index b5a6e976805b..e41e3189286a 100644 --- a/apps/desktop/src/app/chat/composer/model-pill.tsx +++ b/apps/desktop/src/app/chat/composer/model-pill.tsx @@ -1,18 +1,21 @@ import { useStore } from '@nanostores/react' -import { useState } from 'react' +import { useEffect, useState } from 'react' import { useSessionView } from '@/app/chat/session-view' import { ModelMenuCloseContext } from '@/app/shell/model-menu-panel' import { Button } from '@/components/ui/button' import { DropdownMenu, DropdownMenuContent, DropdownMenuTrigger } from '@/components/ui/dropdown-menu' import { GlyphSpinner } from '@/components/ui/glyph-spinner' +import { releaseTypingFocus } from '@/components/ui/keyboard-first' import { Tip } from '@/components/ui/tooltip' import { useI18n } from '@/i18n' import { ChevronDown } from '@/lib/icons' import { formatModelStatusLabel } from '@/lib/model-status-label' import { cn } from '@/lib/utils' -import { $currentModelSource, setModelPickerOpen } from '@/store/session' +import { $currentModelSource, $defaultReasoningEffort, setModelPickerOpen } from '@/store/session' +import { onComposerModelMenuRequest } from './focus' +import { useComposerScope } from './scope' import type { ChatBarState } from './types' const PILL = cn( @@ -48,8 +51,31 @@ export function ModelPill({ const fastMode = useStore(view.$fast) const reasoningEffort = useStore(view.$reasoningEffort) const modelSource = useStore($currentModelSource) + const defaultEffort = useStore($defaultReasoningEffort) const runtimeId = useStore(view.$runtimeId) const [open, setOpen] = useState(false) + const scope = useComposerScope() + const hasLiveMenu = Boolean(model.modelMenuContent) + + // The `composer.modelPicker` hotkey, routed to exactly one surface (the pane + // under the pointer, else the active composer — see requestModelMenuToggle). + // Toggles the live dropdown; with no live menu (gateway closed) it opens the + // full picker dialog, same as clicking the pill. + useEffect( + () => + onComposerModelMenuRequest(target => { + if (target !== scope.target || disabled) { + return + } + + if (hasLiveMenu) { + setOpen(prev => !prev) + } else { + setModelPickerOpen(true) + } + }), + [scope.target, disabled, hasLiveMenu] + ) // The composer pick is sticky: a manual selection is pinned and every NEW // chat uses it instead of the Settings → Model default — silently, which has @@ -68,7 +94,9 @@ export function ModelPill({ ) : ( <> {currentModel.trim() ? ( - {formatModelStatusLabel(currentModel, { fastMode, reasoningEffort })} + + {formatModelStatusLabel(currentModel, { defaultEffort, fastMode, reasoningEffort })} + ) : ( )} @@ -116,8 +144,19 @@ export function ModelPill({ ) } + // Closing the menu ends its claim on the keyboard: Radix restores focus to + // this pill (a toolbar button), so without the release the Enter that + // committed a model also swallows whatever you type next. + const setMenuOpen = (next: boolean) => { + setOpen(next) + + if (!next) { + releaseTypingFocus() + } + } + return ( - + diff --git a/apps/desktop/src/app/chat/composer/rich-editor.test.ts b/apps/desktop/src/app/chat/composer/rich-editor.test.ts index 12e3e9613eff..6c3d2f873326 100644 --- a/apps/desktop/src/app/chat/composer/rich-editor.test.ts +++ b/apps/desktop/src/app/chat/composer/rich-editor.test.ts @@ -4,10 +4,11 @@ import { insertInlineRefsIntoEditor } from './inline-refs' import { composerPlainText, deleteSelectionInEditor, - insertPlainTextAtCaret, + insertComposerContentsAtCaret, normalizeComposerEditorDom, refChipElement, renderComposerContents, + replaceBeforeCaret, RICH_INPUT_SLOT } from './rich-editor' @@ -34,6 +35,89 @@ describe('renderComposerContents', () => { expect(editor.textContent).toContain('raw') expect(composerPlainText(editor)).toBe('@file:`` raw') }) + + it('hydrates a committed leading slash command back to its pill', () => { + // Text-hydration parity with @ refs: a re-render from serialized text + // (draft restore, undo, the trigger commit fallback) must not demote a + // committed no-arg command chip to plain text. + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + + renderComposerContents(editor, '/some-skill @folder:`Desktop` ') + + const pill = editor.querySelector('[data-slash-kind]') + + expect(pill?.getAttribute('data-ref-text')).toBe('/some-skill') + expect(editor.querySelector('[data-ref-kind="folder"]')).not.toBeNull() + expect(composerPlainText(editor)).toBe('/some-skill @folder:`Desktop` ') + }) + + it('keeps a still-typed leading slash token as editable text', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + + // No trailing whitespace — not committed yet. + renderComposerContents(editor, '/some-skil') + + expect(editor.querySelector('[data-slash-kind]')).toBeNull() + expect(composerPlainText(editor)).toBe('/some-skil') + }) + + it('keeps an arg-taking command as text — its tail may be uncommitted prose', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + + renderComposerContents(editor, '/goal ship the redesign') + + expect(editor.querySelector('[data-slash-kind]')).toBeNull() + expect(composerPlainText(editor)).toBe('/goal ship the redesign') + }) +}) + +describe('replaceBeforeCaret across split text nodes', () => { + it('replaces a token that Chromium fragmented into multiple text nodes', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.contentEditable = 'true' + document.body.append(editor) + editor.append(document.createTextNode('see @Desk'), document.createTextNode('top/')) + + const caret = document.createRange() + caret.setStart(editor.lastChild!, 4) + caret.collapse(true) + const selection = window.getSelection()! + selection.removeAllRanges() + selection.addRange(caret) + + const fragment = document.createDocumentFragment() + fragment.append(refChipElement('folder', '`Desktop`'), document.createTextNode(' ')) + + // Token `@Desktop/` (9 chars) spans both text nodes. + expect(replaceBeforeCaret(editor, 9, fragment)).toBe(true) + expect(composerPlainText(editor)).toBe('see @folder:`Desktop` ') + + editor.remove() + }) + + it('refuses when a chip interrupts the span — the token is not contiguous text', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.contentEditable = 'true' + document.body.append(editor) + editor.append(document.createTextNode('a'), refChipElement('file', '`x`'), document.createTextNode('bc')) + + const caret = document.createRange() + caret.setStart(editor.lastChild!, 2) + caret.collapse(true) + const selection = window.getSelection()! + selection.removeAllRanges() + selection.addRange(caret) + + expect(replaceBeforeCaret(editor, 5, document.createDocumentFragment())).toBe(false) + expect(composerPlainText(editor)).toBe('a@file:`x`bc') + + editor.remove() + }) }) describe('normalizeComposerEditorDom', () => { @@ -70,16 +154,40 @@ describe('insertInlineRefsIntoEditor', () => { expect(editor.querySelector(':scope > div')).toBeNull() expect(composerPlainText(editor)).toBe('@file:`src/foo.ts` ') }) + + it('separates a chip from the word the caret sits after', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.append(document.createTextNode('review')) + document.body.append(editor) + caretIn(editor) + + expect(insertInlineRefsIntoEditor(editor, ['@file:`src/a.ts`'])).toBe('review @file:`src/a.ts` ') + + editor.remove() + }) + + it('does not double the space when one is already there', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.append(document.createTextNode('review ')) + document.body.append(editor) + caretIn(editor) + + expect(insertInlineRefsIntoEditor(editor, ['@file:`src/a.ts`'])).toBe('review @file:`src/a.ts` ') + + editor.remove() + }) }) -describe('insertPlainTextAtCaret', () => { +describe('insertComposerContentsAtCaret', () => { it('inserts multiline text as text nodes + br', () => { const editor = document.createElement('div') editor.dataset.slot = RICH_INPUT_SLOT document.body.append(editor) caretIn(editor) - insertPlainTextAtCaret(editor, 'one\ntwo\nthree') + insertComposerContentsAtCaret(editor, 'one\ntwo\nthree') expect(editor.querySelectorAll('br').length).toBe(2) expect(composerPlainText(editor)).toBe('one\ntwo\nthree') @@ -102,12 +210,152 @@ describe('insertPlainTextAtCaret', () => { selection.removeAllRanges() selection.addRange(range) - insertPlainTextAtCaret(editor, 'cd') + insertComposerContentsAtCaret(editor, 'cd') expect(composerPlainText(editor)).toBe('abcdef') editor.remove() }) + + it('lands directives in the text as chips', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + document.body.append(editor) + caretIn(editor) + + insertComposerContentsAtCaret(editor, 'read @url:`https://example.dev/a` now') + + expect(editor.querySelectorAll('[data-ref-kind="url"]').length).toBe(1) + expect(composerPlainText(editor)).toBe('read @url:`https://example.dev/a` now') + + editor.remove() + }) + + // A directive typed by hand chips; the same directive pasted has to chip too, + // or copy/pasting a prompt silently drops every command in it. + it('chips a pasted slash command, including one that ends the paste', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + document.body.append(editor) + caretIn(editor) + + insertComposerContentsAtCaret(editor, '/some-skill') + + expect(editor.querySelector('[data-slash-kind]')?.getAttribute('data-ref-text')).toBe('/some-skill') + // Committed pills carry the trailing space the typed path appends, so a + // later full re-render doesn't read the token as half-typed. + expect(composerPlainText(editor)).toBe('/some-skill ') + + editor.remove() + }) + + it('chips a skill named mid-paste alongside a ref', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + document.body.append(editor) + caretIn(editor) + + insertComposerContentsAtCaret(editor, 'clean @file:`a.ts` with /some-skill then ship') + + expect(editor.querySelectorAll('[data-slash-kind]').length).toBe(1) + expect(editor.querySelectorAll('[data-ref-kind="file"]').length).toBe(1) + expect(composerPlainText(editor)).toBe('clean @file:`a.ts` with /some-skill then ship') + + editor.remove() + }) + + it('leaves a pasted path alone — /usr/local is not a command', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + document.body.append(editor) + caretIn(editor) + + insertComposerContentsAtCaret(editor, 'see /usr/local/bin and /goal ship it') + + expect(editor.querySelector('[data-slash-kind]')).toBeNull() + expect(composerPlainText(editor)).toBe('see /usr/local/bin and /goal ship it') + + editor.remove() + }) + + it('does not chip a command pasted against a word — foo/clean is not a command', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.textContent = 'foo' + document.body.append(editor) + caretIn(editor) + + insertComposerContentsAtCaret(editor, '/some-skill') + + expect(editor.querySelector('[data-slash-kind]')).toBeNull() + expect(composerPlainText(editor)).toBe('foo/some-skill') + + editor.remove() + }) + + it('chips a command pasted right after an existing chip', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.append(refChipElement('file', '`a.ts`')) + document.body.append(editor) + caretIn(editor) + + insertComposerContentsAtCaret(editor, '/some-skill') + + expect(editor.querySelector('[data-slash-kind]')).not.toBeNull() + + editor.remove() + }) +}) + +describe('replaceBeforeCaret', () => { + it('swaps the token before the caret and leaves the caret after the insert', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.textContent = 'see foo' + document.body.append(editor) + + const text = editor.firstChild! + const selection = window.getSelection()! + const range = document.createRange() + + range.setStart(text, 7) + range.collapse(true) + selection.removeAllRanges() + selection.addRange(range) + + const fragment = document.createDocumentFragment() + fragment.append(refChipElement('file', '`src/foo.ts`'), document.createTextNode(' ')) + + expect(replaceBeforeCaret(editor, 3, fragment)).toBe(true) + expect(composerPlainText(editor)).toBe('see @file:`src/foo.ts` ') + expect(selection.getRangeAt(0).collapsed).toBe(true) + + editor.remove() + }) + + it('leaves the editor alone when the caret has no room for the token', () => { + const editor = document.createElement('div') + editor.dataset.slot = RICH_INPUT_SLOT + editor.textContent = 'hi' + document.body.append(editor) + + const selection = window.getSelection()! + const range = document.createRange() + + range.setStart(editor.firstChild!, 2) + range.collapse(true) + selection.removeAllRanges() + selection.addRange(range) + + const fragment = document.createDocumentFragment() + fragment.append(document.createTextNode('x')) + + expect(replaceBeforeCaret(editor, 20, fragment)).toBe(false) + expect(composerPlainText(editor)).toBe('hi') + + editor.remove() + }) }) describe('deleteSelectionInEditor', () => { diff --git a/apps/desktop/src/app/chat/composer/rich-editor.ts b/apps/desktop/src/app/chat/composer/rich-editor.ts index 71491b87496d..a7d8e0406759 100644 --- a/apps/desktop/src/app/chat/composer/rich-editor.ts +++ b/apps/desktop/src/app/chat/composer/rich-editor.ts @@ -7,18 +7,84 @@ * plain-text round-trip. */ import { - DIRECTIVE_CHIP_CLASS, directiveIconElement, directiveIconSvg, formatRefValue, - slashChipClass, + refAttrsHtml, + refChipLabel, type SlashChipKind, slashIconElement } from '@/components/assistant-ui/directive-text' +import { referenceKind, referenceRe } from '@/components/assistant-ui/reference-kinds' + +import { slashCommandMatches, type SlashCommandScanOptions } from './slash-refs' export const RICH_INPUT_SLOT = 'composer-rich-input' -export const REF_RE = /@(file|folder|url|image|tool|line|terminal|session):(`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|\S+)/g +/** Chromium's litter: editing beside a `contenteditable=false` chip splits the + * line and leaves zero-length text nodes behind. They render as nothing and + * serialize as nothing, so no reader of the editor should count them. */ +function isEmptyTextNode(node: ChildNode | null): boolean { + return node?.nodeType === Node.TEXT_NODE && !node.textContent +} + +/** The node before `node`, stepping over that litter. */ +function meaningfulPreviousSibling(node: ChildNode | null): ChildNode | null { + let prev = node?.previousSibling ?? null + + while (isEmptyTextNode(prev)) { + prev = prev?.previousSibling ?? null + } + + return prev +} + +/** The node after `node`, stepping over that litter. */ +function meaningfulNextSibling(node: ChildNode | null): ChildNode | null { + let next = node?.nextSibling ?? null + + while (isEmptyTextNode(next)) { + next = next?.nextSibling ?? null + } + + return next +} + +/** Keep the `data-empty` marker the placeholder paints on in step with the + * editor root's contents. + * + * `:empty` can't be the whole test: a cleared editor keeps a scaffolding
+ * so the contenteditable doesn't collapse, and that break makes `:empty` + * false. Nor can CSS infer it on its own — a text node is invisible to + * selectors, so `one
` and a lone `
` are the same shape, and + * `:has(> br:only-child)` would paint the placeholder over the user's text. + * The code that empties the editor is what knows, so it marks it. + * + * Zero-length text nodes don't count as contents. Chromium leaves them behind + * whenever an edit lands next to a `contenteditable=false` chip, and counting + * them left an editor the user had emptied looking occupied. */ +export function markEditorEmptiness(editor: HTMLElement) { + if (Array.from(editor.childNodes).every(isEmptyTextNode)) { + editor.dataset.empty = '' + } else { + delete editor.dataset.empty + } +} + +/** Drop the marker as IME composition starts, before any preedit text lands. + * + * Input events during composition are deliberately skipped (they carry + * uncommitted preedit text), so nothing else clears the marker until + * `compositionend` — and the hint would otherwise sit behind the hiragana the + * user is composing. `normalizeComposerEditorDom` restores it if composition + * ends with nothing committed. */ +export function beginComposerComposition(editor: HTMLElement) { + delete editor.dataset.empty +} + +/** @see referenceRe — the shared pattern every surface recognises a reference + * with. Module-level `/g` regexes carry `lastIndex`, so call sites reset it. */ +export const REF_RE = referenceRe() const ESC: Record = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' } @@ -34,10 +100,6 @@ export function unquoteRef(raw: string) { return quoted ? raw.slice(1, -1) : raw.replace(/[,.;!?]+$/, '') } -export function refLabel(id: string) { - return id.split(/[\\/]/).filter(Boolean).pop() || id -} - /** Always-quote variant of formatRefValue — chips need a fence even for safe values. */ export function quoteRefValue(value: string) { if (!value.includes('`')) { @@ -59,41 +121,40 @@ export function refChipHtml(kind: string, rawValue: string, displayLabel?: strin const id = unquoteRef(rawValue) const text = `@${kind}:${quoteRefValue(id)}` - return `${directiveIconSvg(kind)}${escapeHtml(displayLabel || refLabel(id))}` + const label = displayLabel || refChipLabel(kind, id) + + return `${directiveIconSvg(kind)}${escapeHtml(label)}` } export function refChipElement(kind: string, rawValue: string, displayLabel?: string) { const id = unquoteRef(rawValue) const text = `@${kind}:${quoteRefValue(id)}` const chip = document.createElement('span') - const label = document.createElement('span') chip.contentEditable = 'false' + chip.title = id chip.dataset.refText = text chip.dataset.refId = id chip.dataset.refKind = kind - chip.className = DIRECTIVE_CHIP_CLASS - label.className = 'truncate' - label.textContent = displayLabel || refLabel(id) - chip.append(directiveIconElement(kind), label) + chip.className = 'ref' + chip.dataset.ref = referenceKind(kind) + chip.append(directiveIconElement(kind), document.createTextNode(displayLabel || refChipLabel(kind, id))) return chip } -/** A non-editable pill for a picked slash command (`/skin nous`, `/tropes`). +/** A non-editable reference for a picked slash command (`/skin nous`, `/tropes`). * `data-ref-text` carries the literal command so `composerPlainText` round-trips * it back to the exact text that gets submitted. */ export function slashChipElement(command: string, kind: SlashChipKind, label?: string) { const chip = document.createElement('span') - const text = document.createElement('span') chip.contentEditable = 'false' chip.dataset.refText = command chip.dataset.slashKind = kind - chip.className = slashChipClass(kind) - text.className = 'truncate' - text.textContent = label || command - chip.append(slashIconElement(kind), text) + chip.className = 'ref' + chip.dataset.ref = kind + chip.append(slashIconElement(kind), document.createTextNode(label || command)) return chip } @@ -112,24 +173,63 @@ function appendTextWithBreaks(target: DocumentFragment | HTMLElement, text: stri }) } -export function appendComposerContents(target: DocumentFragment | HTMLElement, text: string) { +/** Every span of `text` that renders as a chip, in source order. */ +function chipSpans(text: string, options: SlashCommandScanOptions) { + REF_RE.lastIndex = 0 + + const refs = Array.from(text.matchAll(REF_RE)).map(match => { + const start = match.index ?? 0 + + return { end: start + match[0].length, node: () => refChipElement(match[1] || 'file', match[2] || ''), start } + }) + + const commands = slashCommandMatches(text, options).map(match => ({ + end: match.end, + node: () => slashChipElement(match.command, match.kind), + start: match.start + })) + + return [...refs, ...commands].sort((a, b) => a.start - b.start) +} + +/** Build the chip/text DOM for `text`. Directives hydrate back to their pills — + * `@kind:value` refs and `/command` invocations both — so text that arrives + * whole (a paste, a restored draft, an undo step, a rebuilt line) carries the + * same chips the typed path would have committed. */ +export function appendComposerContents( + target: DocumentFragment | HTMLElement, + text: string, + options: SlashCommandScanOptions = {} +) { let cursor = 0 - REF_RE.lastIndex = 0 + for (const span of chipSpans(text, options)) { + // A `@` ref wins an overlap: a command token can't contain an `@`, so the + // only way spans collide is a slash inside a quoted ref value + // (`` @url:`a /clean` ``), which belongs to that value. + if (span.start < cursor) { + continue + } - for (const match of text.matchAll(REF_RE)) { - const index = match.index ?? 0 - appendTextWithBreaks(target, text.slice(cursor, index)) - target.append(refChipElement(match[1] || 'file', match[2] || '')) - cursor = index + match[0].length + appendTextWithBreaks(target, text.slice(cursor, span.start)) + target.append(span.node()) + cursor = span.end } appendTextWithBreaks(target, text.slice(cursor)) } -export function renderComposerContents(target: HTMLElement, text: string) { +export function renderComposerContents(target: HTMLElement, text: string, options?: SlashCommandScanOptions) { target.replaceChildren() - appendComposerContents(target, text) + + // Defaults to live editing, where a token ending the text is still being + // typed (`/wor`) and must stay editable. Callers repainting inert text (a + // restored draft, a sent message opened for edit) pass `trailingCommitted`. + appendComposerContents(target, text, options) + + // The other writer that reshapes the editor root: painting a restored draft + // in clears the marker, clearing back to '' sets it. + markEditorEmptiness(target) } /** Caret range when the selection lives inside `editor`; else null. */ @@ -144,19 +244,95 @@ function composerSelectionRange(editor: HTMLElement) { return { range, selection } } -/** Insert plain text at the caret (replacing any selection). Pastes use this - * instead of `execCommand('insertText')` — Chromium's editing pipeline is - * ~O(n²) on large multiline blobs. */ -export function insertPlainTextAtCaret(editor: HTMLElement, text: string) { +/** Serialized text from the editor's start up to (`container`, `offset`). + * + * Chips are ATOMIC here: each contributes an object-replacement placeholder + * rather than leaking its label text, and a
contributes a newline. That + * makes a chip edge read as a token boundary, which is what both trigger + * detection and directive recognition need. */ +export function serializeTextBefore(editor: HTMLElement, container: Node, offset: number): string { + const probe = document.createRange() + + probe.selectNodeContents(editor) + probe.setEnd(container, offset) + + const scratch = document.createElement('div') + + scratch.append(probe.cloneContents()) + + for (const chip of scratch.querySelectorAll('[data-ref-text]')) { + chip.replaceWith('\uFFFC') + } + + for (const br of scratch.querySelectorAll('br')) { + br.replaceWith('\n') + } + + return scratch.textContent ?? '' +} + +/** True when the insertion point starts a token — the editor's start, or after + * whitespace or a chip. `foo` + a pasted `/clean` is `foo/clean`, not a + * command; `foo ` + the same paste is. */ +function atTokenBoundary(editor: HTMLElement, range: Range | null): boolean { + // No caret means the insert lands at the end, so the question is about the + // editor's last character either way. + const before = range + ? serializeTextBefore(editor, range.startContainer, range.startOffset) + : serializeTextBefore(editor, editor, editor.childNodes.length) + + const last = before.slice(-1) + + return !last || /[\s\uFFFC]/.test(last) +} + +/** Insert text at the caret (replacing any selection), with any directives in + * it landing as chips. Pastes use this instead of `execCommand('insertText')` + * — Chromium's editing pipeline is ~O(n²) on large multiline blobs. + * + * The text arrives whole rather than typed, so a `/command` ending it is + * complete rather than half-written and chips like the rest. + * + * `consumeBefore` characters immediately before the caret are swallowed by the + * insert. That's how a paste into an open `@url:` scope replaces the scope + * instead of stacking on it (`@url:@url:\`https://…\``). */ +export function insertComposerContentsAtCaret(editor: HTMLElement, text: string, consumeBefore = 0) { + const scoped = consumeBefore > 0 ? rangeBeforeCaret(editor, consumeBefore) : null + + if (scoped) { + scoped.deleteContents() + scoped.collapse(true) + + const selection = window.getSelection() + + selection?.removeAllRanges() + selection?.addRange(scoped) + } + const hit = composerSelectionRange(editor) const fragment = document.createDocumentFragment() - appendTextWithBreaks(fragment, text) + // Before measuring the boundary — a replaced selection puts the insertion + // point where the selection started, not where it ended. + if (hit) { + hit.range.deleteContents() + } + + appendComposerContents(fragment, text, { + boundaryBefore: atTokenBoundary(editor, hit?.range ?? null), + trailingCommitted: true + }) + + // A slash pill ending the insert gets the trailing space the typed commit + // path appends, or the next full re-render reads it as a half-typed token + // and demotes it. `@` refs need no marker — REF_RE re-chips them either way. + if ((fragment.lastChild as HTMLElement | null)?.dataset?.slashKind) { + fragment.append(document.createTextNode(' ')) + } const tail = fragment.lastChild if (hit) { - hit.range.deleteContents() hit.range.insertNode(fragment) } else { editor.append(fragment) @@ -172,10 +348,112 @@ export function insertPlainTextAtCaret(editor: HTMLElement, text: string) { } } +/** Range covering exactly `length` serialized characters immediately before a + * collapsed caret, spanning Chromium's split text nodes. Null when the caret + * isn't a collapsed selection in `editor`, or when a chip/
/block boundary + * interrupts before `length` characters are covered — a trigger token is + * always contiguous text, so anything else means "don't touch the DOM here". + * + * This is what keeps chip insertion stable: Chromium fragments text nodes + * around contenteditable=false chips on every edit, so any commit path that + * demands the whole token inside ONE text node (the old check) degrades to a + * full re-render as soon as a chip exists anywhere in the line. */ +export function rangeBeforeCaret(editor: HTMLElement, length: number): Range | null { + const hit = composerSelectionRange(editor) + + if (!hit?.range.collapsed || length <= 0) { + return null + } + + let node: Node | null = hit.range.startContainer + let offset = hit.range.startOffset + + // An element-positioned caret (common right after programmatic caret moves) + // resolves to the end of the text node before it. A chip or
there means + // no text token precedes the caret — bail rather than guess. + if (node.nodeType !== Node.TEXT_NODE) { + node = node.childNodes[offset - 1] ?? null + + if (node?.nodeType !== Node.TEXT_NODE) { + return null + } + + offset = (node.textContent || '').length + } + + let startNode = node as Text + let startOffset = offset + let remaining = length + + while (remaining > 0) { + if (startOffset >= remaining) { + startOffset -= remaining + remaining = 0 + + break + } + + remaining -= startOffset + + const prev: Node | null = startNode.previousSibling + + if (prev?.nodeType !== Node.TEXT_NODE) { + return null + } + + startNode = prev as Text + startOffset = (prev.textContent || '').length + } + + const range = document.createRange() + + range.setStart(startNode, startOffset) + range.setEnd(hit.range.startContainer, hit.range.startOffset) + + return range +} + +/** Swap the `length` characters immediately before a collapsed caret for + * `fragment`, leaving the caret after it. Returns whether it ran. Spans split + * text nodes (see rangeBeforeCaret) — a token typed around existing chips + * still commits in place instead of falling back to a full re-render. */ +export function replaceBeforeCaret(editor: HTMLElement, length: number, fragment: DocumentFragment) { + const range = rangeBeforeCaret(editor, length) + + if (!range) { + return false + } + + const tail = fragment.lastChild + + range.deleteContents() + range.insertNode(fragment) + + if (tail) { + range.setStartAfter(tail) + } + + range.collapse(true) + + const selection = window.getSelection() + + selection?.removeAllRanges() + selection?.addRange(range) + + return true +} + /** Backspace at a collapsed caret immediately after a chip: delete the chip AND * the single trailing space we auto-insert after it, atomically — so removing a * directive never strands an orphaned space (the contenteditable-driven cleanup - * was unreliable). Returns whether it ran. */ + * was unreliable). Returns whether it ran. + * + * "Immediately after" has to be read through Chromium's litter. Committing a + * completion empties the typed token's text node rather than removing it, and + * `Range.insertNode` splits around the caret, so the chip routinely sits + * between zero-length text nodes. Reading those as content made the caret look + * like it was after plain text; the delete declined and Chromium's own + * backspace bounced between the leftovers instead of removing the chip. */ export function deleteChipBeforeCaret(editor: HTMLElement): boolean { const hit = composerSelectionRange(editor) @@ -187,16 +465,20 @@ export function deleteChipBeforeCaret(editor: HTMLElement): boolean { let chip: ChildNode | null = null if (startContainer === editor) { - chip = startOffset > 0 ? editor.childNodes[startOffset - 1] : null + chip = startOffset > 0 ? (editor.childNodes[startOffset - 1] ?? null) : null + + if (isEmptyTextNode(chip)) { + chip = meaningfulPreviousSibling(chip) + } } else if (startContainer.nodeType === Node.TEXT_NODE && startOffset === 0) { - chip = startContainer.previousSibling + chip = meaningfulPreviousSibling(startContainer as ChildNode) } if (chip?.nodeType !== Node.ELEMENT_NODE || !(chip as HTMLElement).dataset.refText) { return false } - const after = chip.nextSibling + const after = meaningfulNextSibling(chip) chip.remove() // Drop the auto-inserted trailing space; keep any real following text. @@ -276,6 +558,15 @@ export function composerPlainText(node: Node): string { return el.dataset.refText } + // An editor holding nothing but the placeholder
is EMPTY. That
is + // scaffolding normalizeComposerEditorDom adds so the contenteditable keeps + // its height — not a line the user typed. Reading it as "\n" is how a + // just-cleared composer stayed non-empty: the newline got stashed as the + // session's draft and painted back on return. + if (el.dataset.slot === RICH_INPUT_SLOT && el.childNodes.length === 1 && el.firstChild?.nodeName === 'BR') { + return '' + } + if (el.tagName === 'BR') { return '\n' } @@ -296,6 +587,106 @@ export function placeCaretEnd(element: HTMLElement) { selection?.addRange(range) } +/** The caret's offset in `composerPlainText` coordinates, so it can be restored + * after the editor is re-rendered from text (undo/redo). A chip counts as its + * whole `@kind:value` text — the same units the snapshot measures. */ +export function caretOffsetInEditor(editor: HTMLElement): number { + const selection = window.getSelection() + const range = selection?.rangeCount ? selection.getRangeAt(0) : null + + if (!range || !editor.contains(range.commonAncestorContainer)) { + return composerPlainText(editor).length + } + + const before = range.cloneRange() + before.selectNodeContents(editor) + before.setEnd(range.startContainer, range.startOffset) + + // The scratch container must carry the editor's slot marker: composerPlainText + // appends a trailing "\n" to any other block element, which would inflate + // every offset by one and land the restored caret a character late. + const container = document.createElement('div') + container.dataset.slot = RICH_INPUT_SLOT + container.append(before.cloneContents()) + + return composerPlainText(container).length +} + +/** Place the caret `offset` characters into the editor, in the same + * `composerPlainText` coordinates `caretOffsetInEditor` reports. Lands after a + * chip it would otherwise split, since a chip is a single atomic unit. */ +export function placeCaretAtOffset(editor: HTMLElement, offset: number) { + const selection = window.getSelection() + + if (!selection) { + return + } + + let remaining = offset + + const walk = (node: Node): Range | null => { + for (const child of Array.from(node.childNodes)) { + if (child.nodeType === Node.TEXT_NODE) { + const length = (child.textContent || '').length + + if (remaining <= length) { + const range = document.createRange() + range.setStart(child, remaining) + range.collapse(true) + + return range + } + + remaining -= length + + continue + } + + if (child.nodeType !== Node.ELEMENT_NODE) { + continue + } + + const el = child as HTMLElement + + // Chips and
are atomic: consume their serialized length whole. + if (el.dataset.refText || el.tagName === 'BR') { + const length = el.dataset.refText ? el.dataset.refText.length : 1 + + if (remaining < length) { + const range = document.createRange() + range.setStartBefore(el) + range.collapse(true) + + return range + } + + remaining -= length + + continue + } + + const hit = walk(el) + + if (hit) { + return hit + } + } + + return null + } + + const range = walk(editor) + + if (range) { + selection.removeAllRanges() + selection.addRange(range) + + return + } + + placeCaretEnd(editor) +} + /** Nothing but a break / whitespace (recursively) — i.e. no real text or chip. */ function isBlankNode(node: ChildNode | null): boolean { if (!node) { @@ -325,6 +716,14 @@ function isBlankNode(node: ChildNode | null): boolean { * rendering emits (we use text nodes +
+ chips). Real
line breaks * (Shift+Enter, which sit after actual text) are preserved. */ export function normalizeComposerEditorDom(editor: HTMLElement) { + // Chromium's zero-length text nodes first: every check below reads siblings, + // and litter between them makes a chip look like it has text either side. + for (const child of Array.from(editor.childNodes)) { + if (isEmptyTextNode(child)) { + child.remove() + } + } + // A trailing block wrapper holding only a break/whitespace is the phantom // "new line" Chromium adds after a chip on backspace — drop it. const tailBlock = editor.lastChild as HTMLElement | null @@ -366,6 +765,9 @@ export function normalizeComposerEditorDom(editor: HTMLElement) { // composer to appear as a tiny dot/pixel. Ensure there's always at least // one
so the element maintains intrinsic height. The CSS min-height // is a belt; the
is suspenders — together they prevent the shrink. + // That break is also why emptiness has to be marked, not inferred. + markEditorEmptiness(editor) + if (editor.childNodes.length === 0) { editor.appendChild(document.createElement('br')) } diff --git a/apps/desktop/src/app/chat/composer/scope.tsx b/apps/desktop/src/app/chat/composer/scope.tsx index 67581a39e30d..e7e7aff880ef 100644 --- a/apps/desktop/src/app/chat/composer/scope.tsx +++ b/apps/desktop/src/app/chat/composer/scope.tsx @@ -23,20 +23,18 @@ export interface ComposerScope { /** This scope's "turn parked on user input" edge — gates Esc-to-stop. */ $awaitingInput: ReadableAtom attachments: ComposerAttachmentScope - /** Only the main scope may pop out (the floating composer is a singleton). */ - popoutAllowed: boolean - /** Imperative read of this scope's transcript (input-history browse) — - * never subscribed, so streaming stays out of the composer's renders. */ - readMessages: () => ChatMessage[] + /** This scope's transcript. Read it imperatively (input-history browse) to + * keep streaming out of the composer's renders; subscribe only off-render + * (auto-speak) where the reply edge is the whole point. */ + $messages: ReadableAtom /** Focus-bus routing key (`'main'` | `'tile:'`). */ target: ComposerTarget } export const MAIN_COMPOSER_SCOPE: ComposerScope = { $awaitingInput: $activeSessionAwaitingInput, + $messages, attachments: mainComposerScope, - popoutAllowed: true, - readMessages: () => $messages.get(), target: 'main' } diff --git a/apps/desktop/src/app/chat/composer/slash-refs.test.ts b/apps/desktop/src/app/chat/composer/slash-refs.test.ts new file mode 100644 index 000000000000..2b50d750875e --- /dev/null +++ b/apps/desktop/src/app/chat/composer/slash-refs.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' + +import { slashCommandMatches } from './slash-refs' + +const commands = (text: string, options?: Parameters[1]) => + slashCommandMatches(text, options).map(match => `${match.kind}:${match.command}`) + +describe('slashCommandMatches', () => { + it('recognizes a leading command and a skill named mid-prose', () => { + expect(commands('/some-skill clean this with /other-skill please')).toEqual([ + 'skill:/some-skill', + 'skill:/other-skill' + ]) + }) + + it('leaves a path alone — /usr/local/bin is not a command', () => { + expect(commands('see /usr/local/bin ')).toEqual([]) + }) + + it('holds a trailing token as still-typed unless the text is inert', () => { + expect(commands('/some-skill')).toEqual([]) + expect(commands('/some-skill', { trailingCommitted: true })).toEqual(['skill:/some-skill']) + }) + + it('leaves an arg-taking command as text — its tail may be prose', () => { + expect(commands('/goal ship the redesign')).toEqual([]) + }) + + it('leaves a command with no desktop surface as text', () => { + expect(commands('/exit now')).toEqual([]) + }) + + it('offers a built-in only as an invocation, never mid-message', () => { + // Mirrors what the popover offers: `/new` acts on the app, so it means + // nothing dropped into a sentence, while a skill reads as "handle this + // part with X". + expect(commands('/new ')).toEqual(['command:/new']) + expect(commands('start over with /new ')).toEqual([]) + expect(commands('start over with /some-skill ')).toEqual(['skill:/some-skill']) + }) + + it('disqualifies a leading token when the text lands mid-word', () => { + expect(commands('/some-skill ', { boundaryBefore: false })).toEqual([]) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/slash-refs.ts b/apps/desktop/src/app/chat/composer/slash-refs.ts new file mode 100644 index 000000000000..db0cf42adb66 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/slash-refs.ts @@ -0,0 +1,105 @@ +/** + * Slash-command recognition for text the composer did not watch being typed — + * a paste, a restored draft, an undo step, a rebuilt line. + * + * The typed path chips a command as it's picked or accepted, so the composer + * agrees with what the sent message renders (`SLASH_SKILL_RE` in + * directive-text). Text that arrives whole never passed through that path, so + * it needs the same commands recognized in place — on exactly the terms the + * typed path would have used, or hydration invents pills the popover would + * never have committed. + */ +import type { SlashChipKind } from '@/components/assistant-ui/directive-text' +import { + desktopSlashCommandArgumentMode, + isDesktopSlashCommand, + resolveDesktopCommand +} from '@/lib/desktop-slash-commands' + +// A command token starts a word and doesn't continue into a path: `/usr/local` +// is a path, not a `/usr` command. Same shape the sent message uses to decide +// what renders as a pill, so the composer and the transcript agree. +const SLASH_COMMAND_RE = /(?<=^|\s)\/([a-zA-Z][\w-]*)(?![\w-]*\/)/g + +export interface SlashCommandMatch { + /** The command with its leading slash, e.g. `/clean`. */ + command: string + end: number + kind: SlashChipKind + start: number +} + +export interface SlashCommandScanOptions { + /** + * Whether the text is preceded by a token boundary. False when it's being + * inserted mid-word (a paste landing against existing characters), which + * disqualifies a token at index 0 — `foo/clean` is not a command. It also + * makes that token mid-message rather than an invocation. + */ + boundaryBefore?: boolean + /** + * Whether a token ending the text counts as committed. True for inert text + * (a paste, dropped content): nothing is being typed, so `/clean` at the end + * is the whole command. False while editing live, where a trailing `/wor` is + * a half-typed query the popover owns and must leave editable. + */ + trailingCommitted?: boolean +} + +/** + * Only commands with NO argument stage chip: their committed pill is exactly + * the bare `/name`, so the boundary is unambiguous. Arg-taking commands + * (`/goal ship it`) stay text — their tail may be prose. Commands with no + * desktop surface at all (`/exit`, `/config`) stay text too. + */ +function chippableKind(command: string): SlashChipKind | null { + if (!isDesktopSlashCommand(command) || desktopSlashCommandArgumentMode(command) !== null) { + return null + } + + return resolveDesktopCommand(command) ? 'command' : 'skill' +} + +/** Every `/command` in `text` that should render as a pill, in source order. */ +export function slashCommandMatches(text: string, options: SlashCommandScanOptions = {}): SlashCommandMatch[] { + const { boundaryBefore = true, trailingCommitted = false } = options + + if (!text.includes('/')) { + return [] + } + + const matches: SlashCommandMatch[] = [] + + for (const match of text.matchAll(SLASH_COMMAND_RE)) { + const start = match.index ?? 0 + const command = match[0] + const end = start + command.length + const after = text[end] + + // A committed pill always carries its auto-inserted trailing space, which + // is what separates it from a token still being typed. + if (after === undefined ? !trailingCommitted : !/\s/.test(after)) { + continue + } + + // Only the FIRST token can be an invocation, and only when the text lands + // on a token boundary — `foo` + a pasted `/clean` is `foo/clean`. + const invocation = start === 0 + + if (invocation && !boundaryBefore) { + continue + } + + const kind = chippableKind(command) + + // Later tokens are references dropped into prose, where the popover offers + // SKILLS alone — a built-in like `/new` acts on the app and means nothing + // mid-sentence. Hydration has to agree, or pasted text grows pills typing + // never would. + if (kind && (invocation || kind === 'skill')) { + matches.push({ command, end, kind, start }) + } + } + + return matches +} diff --git a/apps/desktop/src/app/chat/composer/status-stack/coding-row.test.tsx b/apps/desktop/src/app/chat/composer/status-stack/coding-row.test.tsx new file mode 100644 index 000000000000..326cdfcfd6fb --- /dev/null +++ b/apps/desktop/src/app/chat/composer/status-stack/coding-row.test.tsx @@ -0,0 +1,93 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { atom } from 'nanostores' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { $notifications, clearNotifications } from '@/store/notifications' + +vi.mock('@/store/coding-status', () => ({ + registerRepoStatusCwd: () => undefined, + repoStatusForCwd: () => + atom({ + added: 12, + ahead: 0, + behind: 0, + branch: 'bb/hitbox', + defaultBranch: 'main', + detached: false, + removed: 3, + untracked: 0 + }), + repoWorktreesForCwd: () => atom([]) +})) + +const { CodingStatusRow } = await import('./coding-row') + +describe('CodingStatusRow', () => { + afterEach(() => { + cleanup() + }) + + it('opens the review pane from the branch and the diff counts, never the bar itself', () => { + const onOpen = vi.fn() + + const { container } = render() + + const bar = container.querySelector('.coding-status-bar') + + expect(bar).not.toBeNull() + + fireEvent.click(bar!) + expect(onOpen).not.toHaveBeenCalled() + + fireEvent.click(screen.getByText('bb/hitbox')) + expect(onOpen).toHaveBeenCalledTimes(1) + + fireEvent.click(screen.getByText('12')) + expect(onOpen).toHaveBeenCalledTimes(2) + }) + + it('wraps the click targets without adding a layout box', () => { + const { container } = render( undefined} repoPath="/repo" />) + + // `display: contents` is what keeps the branch label and the counts direct + // flex children of the row — the hit areas cost nothing visually. + expect(screen.getByText('bb/hitbox').parentElement?.classList.contains('contents')).toBe(true) + expect(screen.getByText('12').closest('button')?.classList.contains('contents')).toBe(true) + // The glyph button fills the row's existing 3.5 leading slot exactly. + expect(container.querySelector('button[class~="size-3.5"]')).not.toBeNull() + }) + + it('parks the copy glyph against the end of the path, not the end of the row', () => { + render( undefined} repoPath="/Users/someone/www/repo" />) + + const path = screen.getByText('~/www/repo') + + // The path sizes to its content and the glyph is its immediate sibling, so + // the pair reads as one unit. `flex-1` belongs to the wrapper (which holds + // the row's slack open) — on the label it stretched the text and pushed the + // glyph out to the kebab. + expect(path.classList.contains('flex-1')).toBe(false) + expect(path.parentElement?.classList.contains('flex-1')).toBe(true) + expect(path.nextElementSibling?.tagName).toBe('BUTTON') + }) + + it('copies the absolute cwd inline — checkmark feedback, no toast', async () => { + const writeText = vi.fn().mockResolvedValue(undefined) + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }) + clearNotifications() + + render( undefined} repoPath="/Users/someone/www/repo" />) + + // Painted tildified, copied raw. + expect(screen.getByText('~/www/repo')).toBeTruthy() + + const copy = screen.getByRole('button', { name: 'Copy Path' }) + + fireEvent.click(copy) + + await waitFor(() => expect(writeText).toHaveBeenCalledWith('/Users/someone/www/repo')) + // Confirmation is the button turning into a checkmark, not a notification. + await waitFor(() => expect(screen.getByRole('button', { name: 'Copied' })).toBeTruthy()) + expect($notifications.get()).toHaveLength(0) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx b/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx index aee735e4aea4..842b1ac37ec3 100644 --- a/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx +++ b/apps/desktop/src/app/chat/composer/status-stack/coding-row.tsx @@ -3,20 +3,21 @@ import { memo, useEffect, useRef, useState } from 'react' import { WorktreeDialog } from '@/app/chat/sidebar/projects/worktree-dialog' import { StatusRow } from '@/components/chat/status-row' +import { + type ActionItemSpec, + ActionsContextMenu, + ActionsMenu, + type MenuKit, + renderActionItem +} from '@/components/ui/actions-menu' import { Button } from '@/components/ui/button' import { Codicon } from '@/components/ui/codicon' +import { CopyButton } from '@/components/ui/copy-button' import { DiffCount } from '@/components/ui/diff-count' -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger -} from '@/components/ui/dropdown-menu' import type { HermesGitBranch } from '@/global' import { useI18n } from '@/i18n' -import { $repoStatus, $repoWorktrees } from '@/store/coding-status' +import { displayPath } from '@/lib/display-path' +import { registerRepoStatusCwd, repoStatusForCwd, repoWorktreesForCwd } from '@/store/coding-status' import { notifyError } from '@/store/notifications' import { $newWorktreeRequest } from '@/store/projects' @@ -63,14 +64,25 @@ export const CodingStatusRow = memo(function CodingStatusRow({ const { t } = useI18n() const s = t.statusStack.coding const p = t.sidebar.projects - const status = useStore($repoStatus) - const worktrees = useStore($repoWorktrees) + const fileMenu = t.fileMenu + const resolvedRepoPath = repoPath?.trim() || undefined + // This surface's OWN worktree, always — never the primary's. The row used to + // fall back to the global `$repoStatus` for a blank repoPath, which painted + // the main pane's branch/± onto a tile whose cwd hadn't resolved yet. That + // fallback bought nothing (the primary's computed is keyed to `$currentCwd`, + // which is blank in exactly the same case) and cost a wrong-tree rail. + const status = useStore(repoStatusForCwd(resolvedRepoPath)) + const worktrees = useStore(repoWorktreesForCwd(resolvedRepoPath)) + + // While mounted, keep this worktree in the coding-status refresh set so the + // turn-settle / tool-complete / focus edges re-probe it too (tiles otherwise + // only refreshed when the MAIN cwd probe happened to cover them). + useEffect(() => registerRepoStatusCwd(resolvedRepoPath), [resolvedRepoPath]) // Shared worktree dialog — replaces the old inline dialog. Opened by the // dropdown menu's "branch off" items and the global ⌘⇧B hotkey. const [worktreeOpen, setWorktreeOpen] = useState(false) const [worktreeBase, setWorktreeBase] = useState(undefined) - const resolvedRepoPath = repoPath?.trim() || undefined const switchToBranch = async (branch: string) => { if (!onSwitchBranch) { @@ -91,6 +103,7 @@ export const CodingStatusRow = memo(function CodingStatusRow({ const worktreeReq = useStore($newWorktreeRequest) const lastWorktreeReqRef = useRef(worktreeReq) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (worktreeReq === lastWorktreeReqRef.current) { return @@ -152,122 +165,174 @@ export const CodingStatusRow = memo(function CodingStatusRow({ // they're the only change (otherwise +/- tells the story). const untrackedOnly = !hasLineDelta && status.untracked > 0 + // The branch actions, rendered identically by the kebab dropdown and the + // row's right-click menu so the two never drift. `onBranchOff` gates the + // whole menu (omitted = remote backend), matching the kebab. + const renderBranchItems = (kit: MenuKit) => { + const branchItems: ActionItemSpec[] = branchTargets.map(target => ({ + key: target.base ?? '__head__', + label: {target.label}, + onSelect: () => startBranch(target.base) + })) + + const worktreeItems: ActionItemSpec[] = otherWorktrees.map(worktree => ({ + key: worktree.path, + label: {worktree.branch}, + onSelect: () => onOpenWorktree?.(worktree.path) + })) + + return ( + <> + {s.newBranch} + {branchItems.map(item => renderActionItem(kit, item))} + {switchTarget && + renderActionItem(kit, { + key: '__switch__', + label: {s.switchTo(switchTarget)}, + onSelect: () => void switchToBranch(switchTarget) + })} + + {s.worktrees} + {worktreeItems.map(item => renderActionItem(kit, item))} + {/* Create a fresh worktree off the current HEAD (the generic "spin up a + worktree here", mirroring the sidebar's + button). */} + {renderActionItem(kit, { + key: '__start__', + label: {p.startWork}, + onSelect: () => startBranch(undefined) + })} + {onConvertBranch && + renderActionItem(kit, { + key: '__convert__', + label: {p.convertBranch}, + onSelect: () => startBranch(undefined) + })} + + ) + } + return ( <> - } - onActivate={onOpen} - > -
- - {branchLabel} - + + + + + } + > +
+ {/* Branch name — the other half of the review-pane target. `contents` + so the button lays out nothing of its own: the label stays the + same flex child it always was, and the hit area is the text. */} + - {/* Branch actions kebab — same pattern as the session/worktree rows. - ALWAYS laid out; only its opacity flips on hover/focus/open, so - revealing it never reflows the row (no layout shift). pointer-events - follow opacity so the invisible trigger isn't clickable at rest. */} - {onBranchOff && ( - - + {/* Worktree path + copy — plain muted text, not a chip. Always in the + flex so hover doesn't reflow the row; opacity alone reveals the + pair. The path sizes to its content (the `flex-1` lives on the + wrapper) so the glyph sits against the end of the text instead of + drifting to the far edge of the row. `displayPath` collapses + home → ~; the copy still takes the real absolute path, and it's + the shared `CopyButton` so it confirms with the same inline + checkmark as every other copy in the app. */} + {resolvedRepoPath && ( +
+ + {displayPath(resolvedRepoPath)} + + +
+ )} + + {/* Branch actions kebab — same pattern as the session/worktree rows. + ALWAYS laid out; only its opacity flips on hover/focus/open, so + revealing it never reflows the row (no layout shift). pointer-events + follow opacity so the invisible trigger isn't clickable at rest. */} + {onBranchOff && ( + - - {/* The row sits at the bottom of the screen (above the composer), - so the menu opens upward. */} - - {s.newBranch} - {branchTargets.map(target => ( - startBranch(target.base)}> - {target.label} - - ))} + + )} +
- {switchTarget && ( - void switchToBranch(switchTarget)}> - {s.switchTo(switchTarget)} - - )} + {/* The counts describe what's in the review pane, so clicking them + opens it. `contents` again: the two spans stay direct flex children + of the row, keeping their gap and `ml-auto` behaviour untouched. */} + {(status.ahead > 0 || status.behind > 0 || hasLineDelta || untrackedOnly) && ( + )} -
- - {(status.ahead > 0 || status.behind > 0) && ( - - {status.ahead > 0 && ( - - ↑ - {status.ahead} - - )} - {status.behind > 0 && ( - - ↓ - {status.behind} - - )} - - )} - - {hasLineDelta ? ( - - ) : untrackedOnly ? ( - - {s.changed(status.untracked)} - - ) : null} - + + {resolvedRepoPath && onOpenWorktree && ( ({ + detail, + status, + title, + updatedAt: Date.now() +}) + +function renderStack(sessionId: null | string = SID) { + return render( + + + + + + ) +} + +describe('ComposerStatusStack goal indicator', () => { + beforeEach(() => { + $goalsBySession.set({}) + }) + + afterEach(() => { + cleanup() + $goalsBySession.set({}) + }) + + it('renders nothing when the session has no goal', () => { + const view = renderStack() + + expect(view.container.firstChild).toBeNull() + }) + + it('shows an active goal with its title', () => { + $goalsBySession.set({ [SID]: goal('active') }) + + renderStack() + + expect(screen.getByText('Goal active')).toBeTruthy() + expect(screen.getByText('ship the feature')).toBeTruthy() + }) + + it('labels a paused goal as paused', () => { + $goalsBySession.set({ [SID]: goal('paused') }) + + renderStack() + + expect(screen.getByText('Goal paused')).toBeTruthy() + expect(screen.getByText('ship the feature')).toBeTruthy() + }) + + it('shows the continuation detail line for an active goal', () => { + $goalsBySession.set({ [SID]: goal('active', 'ship it', 'Continuing toward goal (3/20)') }) + + renderStack() + + expect(screen.getByText('Continuing toward goal (3/20)')).toBeTruthy() + }) + + it('scopes the indicator to the goal-owning session', () => { + $goalsBySession.set({ 'other-session': goal('active') }) + + const view = renderStack() + + expect(view.container.firstChild).toBeNull() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/status-stack/index.tsx b/apps/desktop/src/app/chat/composer/status-stack/index.tsx index 739080be8b85..de86ba14d0fd 100644 --- a/apps/desktop/src/app/chat/composer/status-stack/index.tsx +++ b/apps/desktop/src/app/chat/composer/status-stack/index.tsx @@ -1,6 +1,6 @@ import { useStore } from '@nanostores/react' -import { type ReactNode, useEffect, useLayoutEffect, useMemo, useRef } from 'react' -import { useNavigate } from 'react-router-dom' +import { type ReactNode, useEffect, useMemo } from 'react' +import { useNavigate } from 'react-router' import { blurComposerInput } from '@/app/chat/composer/focus' import { AGENTS_ROUTE } from '@/app/routes' @@ -11,6 +11,7 @@ import { Button } from '@/components/ui/button' import { Codicon } from '@/components/ui/codicon' import { Tip, TipKeybindLabel } from '@/components/ui/tooltip' import { type Translations, useI18n } from '@/i18n' +import { useSessionSlice } from '@/lib/use-session-slice' import { cn } from '@/lib/utils' import { $billingBlock } from '@/store/billing-block' import { @@ -22,6 +23,7 @@ import { type StatusGroup, stopBackgroundProcess } from '@/store/composer-status' +import { refreshSessionGoal } from '@/store/goals' import { $previewStatusBySession, dismissPreviewArtifact } from '@/store/preview-status' import { $threadScrolledUp } from '@/store/thread-scroll' import { openSessionInNewWindow } from '@/store/windows' @@ -41,12 +43,25 @@ const isLocalhostPreview = (target: string): boolean => /\b(?:localhost|127\.0\. // Real codicons per group (no sparkles): a checklist for todos, the agent glyph // for subagents, a background process glyph for background tasks. const GROUP_ICON: Record = { + goal: 'target', todo: 'checklist', subagent: 'agent', background: 'server-process' } const groupLabel = (group: StatusGroup, s: Translations['statusStack']) => { + if (group.type === 'goal') { + const status = group.items[0]?.goalStatus + + return status === 'paused' + ? s.goalPaused + : status === 'waiting' + ? s.goalWaiting + : status === 'done' + ? s.goalDone + : s.goalActive + } + if (group.type === 'todo') { return s.todos(group.items.filter(i => i.todoStatus === 'completed').length, group.items.length) } @@ -69,23 +84,25 @@ interface ComposerStatusStackProps { export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackProps) { const { t } = useI18n() const navigate = useNavigate() - const itemsBySession = useStore($statusItemsBySession) - const previewsBySession = useStore($previewStatusBySession) + // Subscribe to THIS session's slice only. Both maps churn on other + // sessions' activity (subagent ticks, background polls, preview updates in + // any tile); a whole-map `useStore` re-rendered every mounted stack — one + // per open tile — on all of it. The per-key arrays are referentially stable + // across unrelated writes, so the slice hook bails out unless OUR session's + // items actually changed. + const items = useSessionSlice($statusItemsBySession, sessionId) + const previews = useSessionSlice($previewStatusBySession, sessionId) const scrolledUp = useStore($threadScrolledUp) const billing = useStore($billingBlock) - const groups = useMemo( - () => groupStatusItems(sessionId ? (itemsBySession[sessionId] ?? []) : []), - [itemsBySession, sessionId] - ) - - const previews = sessionId ? (previewsBySession[sessionId] ?? []) : [] + const groups = useMemo(() => groupStatusItems(items), [items]) // Seed from the registry on session open; event-driven refreshes (terminal / // process tool completions) live in use-message-stream. useEffect(() => { if (sessionId) { void refreshBackgroundProcesses(sessionId) + void refreshSessionGoal(sessionId) } }, [sessionId]) @@ -153,7 +170,7 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro ) : undefined } - defaultCollapsed={group.type !== 'todo'} + defaultCollapsed={group.type !== 'todo' && group.type !== 'goal'} icon={} label={groupLabel(group, t.statusStack)} > @@ -190,43 +207,15 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro sections.push({ key: 'queue', node: queue }) } + // Micro actions are the TOP-MOST thing in the whole overlay lane — above the + // status card, above the billing wall, above everything. They're the only + // rows up here you press instead of read, so nothing may ever stack on top + // of them. Rendered outside the card (below) so the pills float. const visible = sections.length > 0 - const stackRef = useRef(null) - - // The stack is out of flow (overlays the thread), so the composer's measured - // height never sees it. Publish our own measured height — bucketed like the - // composer's, to avoid style invalidation churn — so the thread's - // last-message clearance can add it and the stack never hides messages. - useLayoutEffect(() => { - const root = document.documentElement - const el = stackRef.current - - if (!visible || !el) { - root.style.removeProperty('--status-stack-measured-height') - - return - } - let last = -1 - - const sync = () => { - const bucket = Math.round(el.getBoundingClientRect().height / 8) * 8 - - if (bucket !== last) { - last = bucket - root.style.setProperty('--status-stack-measured-height', `${bucket}px`) - } - } - - const observer = new ResizeObserver(sync) - observer.observe(el) - sync() - - return () => { - observer.disconnect() - root.style.removeProperty('--status-stack-measured-height') - } - }, [visible]) + // No height to publish: the stack is an in-flow child of the composer dock, + // so the dock's own measurement (--composer-measured-height) already covers + // it and the thread clears both with one number. if (!visible) { return null @@ -234,12 +223,11 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro return (
blurComposerInput()} - ref={stackRef} > {/* The card paints the shared --composer-fill (rest / scrolled / focused all match the composer surface by construction); on scroll we only @@ -247,20 +235,22 @@ export function ComposerStatusStack({ queue, sessionId }: ComposerStatusStackPro Rounded top, square bottom; the bottom border is TRANSPARENT — the composer surface's visible top border (which sits at a higher z) is the single shared seam, so the two read as one fused capsule. */} -
- {sections.map(section => ( -
{section.node}
- ))} -
+ {sections.length > 0 && ( +
+ {sections.map(section => ( +
{section.node}
+ ))} +
+ )}
) } diff --git a/apps/desktop/src/app/chat/composer/status-stack/preview-row.tsx b/apps/desktop/src/app/chat/composer/status-stack/preview-row.tsx index cf721d2ae936..dc40c31de2e2 100644 --- a/apps/desktop/src/app/chat/composer/status-stack/preview-row.tsx +++ b/apps/desktop/src/app/chat/composer/status-stack/preview-row.tsx @@ -11,7 +11,7 @@ import { cn } from '@/lib/utils' import { PREVIEW_PANE_ID } from '@/store/layout' import { notifyError } from '@/store/notifications' import { $paneOpen } from '@/store/panes' -import { $previewTarget, dismissPreviewTarget, setCurrentSessionPreviewTarget } from '@/store/preview' +import { $previewTabSources, closePreviewForSource, openPreview } from '@/store/preview' import { type PreviewArtifact } from '@/store/preview-status' interface PreviewStatusRowProps { @@ -22,10 +22,10 @@ interface PreviewStatusRowProps { /** One detected artifact, single line, always visible: filename + open + close. */ export const PreviewStatusRow = memo(function PreviewStatusRow({ item, onDismiss }: PreviewStatusRowProps) { const { t } = useI18n() - const activePreview = useStore($previewTarget) + const openSources = useStore($previewTabSources) const previewPaneOpen = useStore($paneOpen(PREVIEW_PANE_ID)) const [opening, setOpening] = useState(false) - const isOpen = activePreview?.source === item.target && previewPaneOpen + const isOpen = openSources.includes(item.target) && previewPaneOpen const resolveTarget = async () => { const target = await normalizeOrLocalPreviewTarget(item.target, item.cwd || undefined) @@ -43,7 +43,7 @@ export const PreviewStatusRow = memo(function PreviewStatusRow({ item, onDismiss } if (isOpen) { - dismissPreviewTarget() + closePreviewForSource(item.target) return } @@ -51,7 +51,7 @@ export const PreviewStatusRow = memo(function PreviewStatusRow({ item, onDismiss setOpening(true) try { - setCurrentSessionPreviewTarget(await resolveTarget(), 'tool-result', item.target) + openPreview(await resolveTarget(), 'tool-result') } catch (error) { notifyError(error, t.preview.unavailable) } finally { diff --git a/apps/desktop/src/app/chat/composer/status-stack/status-row.tsx b/apps/desktop/src/app/chat/composer/status-stack/status-row.tsx index 6857be46ccfe..2c43743c40c5 100644 --- a/apps/desktop/src/app/chat/composer/status-stack/status-row.tsx +++ b/apps/desktop/src/app/chat/composer/status-stack/status-row.tsx @@ -25,6 +25,24 @@ const TODO_GLYPHS: Record, { icon // Left slot: braille spinner while running, otherwise a small status dot // (green = done, red = failed) so the slot is always filled and rows align. function leadingGlyph(item: ComposerStatusItem, s: Translations['statusStack']): ReactNode { + if (item.type === 'goal') { + if (item.goalStatus === 'paused') { + return + } + + if (item.goalStatus === 'done') { + return + } + + return ( + + ) + } + if (item.todoStatus === 'pending') { return ( )} + {item.type === 'goal' && item.currentTool && ( + + {item.currentTool} + + )} {failed && typeof item.exitCode === 'number' && item.exitCode !== 0 && ( {s.exit(item.exitCode)} diff --git a/apps/desktop/src/app/chat/composer/text-utils.test.ts b/apps/desktop/src/app/chat/composer/text-utils.test.ts index 6c6a20780f64..ffca3ba64227 100644 --- a/apps/desktop/src/app/chat/composer/text-utils.test.ts +++ b/apps/desktop/src/app/chat/composer/text-utils.test.ts @@ -4,19 +4,19 @@ import { blobDedupeKey, detectTrigger, extractClipboardImageBlobs } from './text describe('detectTrigger', () => { it('detects a bare slash trigger with an empty query', () => { - expect(detectTrigger('/')).toEqual({ kind: '/', query: '', tokenLength: 1 }) + expect(detectTrigger('/')).toEqual({ kind: '/', query: '', tokenLength: 1, value: '' }) }) it('detects a slash command query', () => { - expect(detectTrigger('/skill')).toEqual({ kind: '/', query: 'skill', tokenLength: 6 }) + expect(detectTrigger('/skill')).toEqual({ kind: '/', query: 'skill', tokenLength: 6, value: 'skill' }) }) it('detects a bare at-mention trigger with an empty query', () => { - expect(detectTrigger('@')).toEqual({ kind: '@', query: '', tokenLength: 1 }) + expect(detectTrigger('@')).toEqual({ kind: '@', query: '', tokenLength: 1, value: '' }) }) it('detects an at-mention query', () => { - expect(detectTrigger('@file')).toEqual({ kind: '@', query: 'file', tokenLength: 5 }) + expect(detectTrigger('@file')).toEqual({ kind: '@', query: 'file', tokenLength: 5, value: 'file' }) }) it('returns null for plain text', () => { @@ -27,31 +27,142 @@ describe('detectTrigger', () => { expect(detectTrigger('/personality ')).toEqual({ kind: '/', query: 'personality ', - tokenLength: 13 + tokenLength: 13, + value: 'personality ' }) expect(detectTrigger('/personality alic')).toEqual({ kind: '/', query: 'personality alic', - tokenLength: 17 + tokenLength: 17, + value: 'personality alic' }) expect(detectTrigger('/tools enable foo')).toEqual({ kind: '/', query: 'tools enable foo', - tokenLength: 17 + tokenLength: 17, + value: 'tools enable foo' }) }) it('does not treat file-style paths as slash triggers', () => { expect(detectTrigger('src/foo/bar')).toBeNull() expect(detectTrigger('/path/to/file')).toBeNull() + // Mid-message paths stay excluded too: a path keeps going past the command + // token, so the trailing-anchored inline trigger never matches it. + expect(detectTrigger('check src/foo/bar')).toBeNull() + expect(detectTrigger('look at /usr/local/bin')).toBeNull() + expect(detectTrigger('and/or')).toBeNull() }) - it('does not trigger slash popover mid-message', () => { - expect(detectTrigger('hello /')).toBeNull() - expect(detectTrigger('hello /skill')).toBeNull() + it('keeps the at-mention live while walking into subfolders', () => { + // A `/` inside the query is path navigation, not the end of the token — + // the popover has to stay open so the next directory level can load. + expect(detectTrigger('@./')).toEqual({ kind: '@', query: './', tokenLength: 3, value: './' }) + expect(detectTrigger('@./src')).toEqual({ kind: '@', query: './src', tokenLength: 6, value: './src' }) + expect(detectTrigger('@~/Desktop/')).toEqual({ + kind: '@', + query: '~/Desktop/', + tokenLength: 11, + value: '~/Desktop/' + }) + expect(detectTrigger('@/usr/local')).toEqual({ + kind: '@', + query: '/usr/local', + tokenLength: 11, + value: '/usr/local' + }) + expect(detectTrigger('@apps/desktop/src')).toEqual({ + kind: '@', + query: 'apps/desktop/src', + tokenLength: 17, + value: 'apps/desktop/src' + }) + }) + + it('treats a chip edge as a token boundary, like whitespace', () => { + // U+FFFC is textBeforeCaret's placeholder for a committed pill. Upstream + // assistant-ui's Lexical DirectivePlugin gets the same semantics from node + // boundaries: typing a trigger right after a chip (no space) still opens + // the popover, and a chip inside a token ends it. + expect(detectTrigger('\uFFFC@Desk')).toEqual({ kind: '@', query: 'Desk', tokenLength: 5, value: 'Desk' }) + // Not position 0, so it's an inline reference — not a command invocation. + expect(detectTrigger('\uFFFC/cle')).toEqual({ + inline: true, + kind: '/', + query: 'cle', + tokenLength: 4, + value: 'cle' + }) + // The placeholder itself never leaks into a query. + expect(detectTrigger('@a\uFFFCb')).toBeNull() + }) + + it('splits a typed ref kind off as the browse scope', () => { + // `@folder:apps/` is ONE token with TWO parts. The kind is the mode the + // user is browsing in, so it's held as `scope` rather than left in `value` + // for every consumer to re-parse (or, worse, to preserve by hand). + expect(detectTrigger('@file:src/main.tsx')).toEqual({ + kind: '@', + query: 'file:src/main.tsx', + scope: 'file', + tokenLength: 18, + value: 'src/main.tsx' + }) + expect(detectTrigger('@folder:apps/')).toEqual({ + kind: '@', + query: 'folder:apps/', + scope: 'folder', + tokenLength: 13, + value: 'apps/' + }) + // A scope with nothing typed after it is the empty-browse state the + // popover renders a header for. + expect(detectTrigger('@url:')).toEqual({ kind: '@', query: 'url:', scope: 'url', tokenLength: 5, value: '' }) + }) + + it('only treats a KNOWN kind as a scope', () => { + // `@teknium1:` is a handle with a colon, not a directive — inventing a + // scope for it would make Backspace eat the whole word. + expect(detectTrigger('@teknium1:')?.scope).toBeUndefined() + expect(detectTrigger('@teknium1:')?.value).toBe('teknium1:') + expect(detectTrigger('@localhost:8080')?.scope).toBeUndefined() + }) + + it('still ends the at-mention token at whitespace', () => { + // The token is whitespace-delimited; a path doesn't change that. + expect(detectTrigger('@./src and more')).toBeNull() + expect(detectTrigger('look at @apps/desktop')).toEqual({ + kind: '@', + query: 'apps/desktop', + tokenLength: 13, + value: 'apps/desktop' + }) + }) + + it('treats a mid-message slash as an inline reference', () => { + // Skills have to be reachable anywhere in a prompt, not just at position 0. + expect(detectTrigger('hello /')).toEqual({ kind: '/', inline: true, query: '', tokenLength: 1, value: '' }) + expect(detectTrigger('hello /clean')).toEqual({ + kind: '/', + inline: true, + query: 'clean', + tokenLength: 6, + value: 'clean' + }) + expect(detectTrigger('text\n/skill')).toEqual({ + kind: '/', + inline: true, + query: 'skill', + tokenLength: 6, + value: 'skill' + }) + }) + + it('does not carry arg completion into an inline slash reference', () => { + // Only a position-0 slash is a real invocation, so `/personality alic` + // mid-message is prose — the trigger ends at the command token. expect(detectTrigger('hello there /personality alic')).toBeNull() - expect(detectTrigger('text\n/skill')).toBeNull() - expect(detectTrigger('multi word message /')).toBeNull() + expect(detectTrigger('run /tools enable foo')).toBeNull() }) it('still anchors at-mention triggers strictly at the token edge', () => { @@ -101,6 +212,47 @@ describe('extractClipboardImageBlobs', () => { expect(extractClipboardImageBlobs(clipboard)).toEqual([image]) }) + + // A rich-text copy (Discord thread, web page, doc) carries prose plus whatever + // inline images the page decorated it with. That is a TEXT paste: attaching the + // page's placeholder graphics as composer images while the text vanished is the + // "blank attachments, no message" bug. + it('ignores inline HTML images when the copy carries its own text', () => { + const clipboard = { + files: { length: 0, item: () => null }, + getData: (type: string) => + type === 'text/html' + ? `

hello from the thread

` + : 'hello from the thread', + items: [] + } as unknown as DataTransfer + + expect(extractClipboardImageBlobs(clipboard)).toEqual([]) + }) + + it('keeps inline HTML images when the copy is image-only', () => { + const clipboard = { + files: { length: 0, item: () => null }, + getData: (type: string) => + type === 'text/html' ? `` : '', + items: [] + } as unknown as DataTransfer + + const blobs = extractClipboardImageBlobs(clipboard) + + expect(blobs).toHaveLength(1) + expect(blobs[0]?.type).toBe('image/png') + }) + + it('drops sub-thumbnail inline images — spacers, trackers, blurhash placeholders', () => { + const clipboard = { + files: { length: 0, item: () => null }, + getData: (type: string) => (type === 'text/html' ? `` : ''), + items: [] + } as unknown as DataTransfer + + expect(extractClipboardImageBlobs(clipboard)).toEqual([]) + }) }) describe('blobDedupeKey', () => { diff --git a/apps/desktop/src/app/chat/composer/text-utils.ts b/apps/desktop/src/app/chat/composer/text-utils.ts index b9b6adc07f1e..42af807c3927 100644 --- a/apps/desktop/src/app/chat/composer/text-utils.ts +++ b/apps/desktop/src/app/chat/composer/text-utils.ts @@ -1,22 +1,79 @@ import { DATA_IMAGE_URL_RE, dataUrlToBlob } from '@/lib/embedded-images' +import { $reactionsEnabled } from '@/store/reactions-enabled' + +import { serializeTextBefore } from './rich-editor' export interface TriggerState { - kind: '@' | '/' + /** True for a `/` typed mid-message — an inline skill/command reference in + * prose rather than a command invocation. Arg completion doesn't apply. */ + inline?: boolean + kind: '@' | '/' | ':' query: string + /** The `@kind:` prefix the user scoped the browse to, when there is one. */ + scope?: DirectiveScope tokenLength: number + /** `query` minus the `scope:` prefix — the value actually being typed. */ + value: string } +/** Directive kinds the `@` popover can scope a browse to. Mirrors the starter + * rows in use-at-completions and the gateway's `complete.path` prefixes. */ +export const DIRECTIVE_SCOPES = ['file', 'folder', 'url', 'image', 'tool', 'git'] as const + +export type DirectiveScope = (typeof DIRECTIVE_SCOPES)[number] + +// Picking "attach a folder" types `@folder:` into the editor, and everything +// after it is the value being browsed. Parsing that prefix off the query is +// what lets the rest of the composer treat it as the BROWSE MODE it is rather +// than characters the user has to maintain by hand — Tab-descending has to +// carry it down, Backspace has to drop it whole, and a chip landing on it has +// to consume it. +const AT_SCOPE_RE = new RegExp(`^(${DIRECTIVE_SCOPES.join('|')}):(.*)$`) + // `@` triggers stop at the first whitespace — `@file:path` and `@diff` are -// single tokens. `/` triggers keep going so the popover stays live while the -// user types args (`/personality alic` → arg completer suggests `alice`). -// Restricting the slash command name to `[a-zA-Z][\w-]*` avoids matching file -// paths like `src/foo/bar`. +// single tokens, and a path is part of that token: `@./src/`, `@~/Desktop/`, +// and `@file:src/foo` all have to keep the popover live while the user walks +// into subdirectories. Excluding `/` from the query class would end the token +// at the first separator, which is exactly the "can't browse into a folder" +// bug. Restricting the slash command name to `[a-zA-Z][\w-]*` avoids +// matching file paths like `src/foo/bar`. +// +// `/` triggers fire in two shapes, because a slash means two different things +// depending on where it sits: +// +// - At position 0 it's a COMMAND invocation the app executes (SLASH_COMMAND_RE +// is `^`-anchored, and so is the backend's). The popover stays live past the +// command name so arg completion works (`/personality alic` → `alice`). +// - After whitespace it's an inline REFERENCE the user is dropping into prose +// ("clean this up with /clean"). The text submits as an ordinary message, so +// there are no args to complete — the trigger is a single token that ends at +// the next space, exactly like `@`. // -// Slash commands only execute at the beginning of a message, so the `/` -// trigger is anchored strictly at position 0 — not after whitespace — to -// avoid opening the popover mid-message (e.g. `hello /`). -const AT_TRIGGER_RE = /(?:^|[\s])(@)([^\s@/]*)$/ -const SLASH_TRIGGER_RE = /^(\/)((?:[a-zA-Z][\w-]*(?:\s+\S*)*)?)$/ +// Only the FIRST slash can be an invocation, so the inline shape is tested +// first: the command regex's argument tail (`(?:\s+\S*)*`) happily swallows a +// later `/skill` as if it were an argument, which killed completion for every +// slash after a leading command (`/work /cle` → nothing). +// +// The inline shape is what makes skills reachable anywhere in a prompt. Both +// shapes need the trailing `$`: detection runs against the text BEFORE the +// caret, so the match must end where the user is typing. +// +// U+FFFC is the placeholder textBeforeCaret emits for a committed chip. A chip +// edge is a token boundary just like whitespace (upstream assistant-ui's +// Lexical DirectivePlugin gets the same semantics from node boundaries), so +// `@` or `/` typed immediately after a pill still opens the popover. +const AT_TRIGGER_RE = /(?:^|[\s\uFFFC])(@)([^\s@\uFFFC]*)$/ +const SLASH_COMMAND_TRIGGER_RE = /^(\/)((?:[a-zA-Z][\w-]*(?:\s+\S*)*)?)$/ +const SLASH_INLINE_TRIGGER_RE = /[\s\uFFFC](\/)([a-zA-Z][\w-]*)?$/ +// `:joy` → emoji completions, Slack-style. Boundary-anchored so a mid-word +// colon (`localhost:8080`, `note:`) never fires; two chars minimum so a bare +// `:` or `:D` smiley doesn't open a popover the user didn't ask for. +const EMOJI_TRIGGER_RE = /(?:^|[\s\uFFFC])(:)([a-zA-Z0-9_+-]{2,})$/ + +const INLINE_IMAGE_SRC_RE = /]*?\bsrc\s*=\s*["'](data:image\/[^"']+)["']/gi +// Below this, an inline data URL is chrome rather than content — a spacer, a +// 1×1 tracker, or a blurhash placeholder. Real pasted artwork clears it easily. +const MIN_INLINE_IMAGE_BYTES = 4096 /** Stable key for paste dedupe — `items` and `files` often mirror the same image as different objects. */ export function blobDedupeKey(blob: Blob): string { @@ -73,16 +130,22 @@ export function extractClipboardImageBlobs(clipboard: DataTransfer): Blob[] { if (DATA_IMAGE_URL_RE.test(text)) { push(dataUrlToBlob(text)) - } - if (blobs.length === 0) { - const html = clipboard.getData('text/html') - - if (html) { - const matches = html.matchAll(/]*?\bsrc\s*=\s*["'](data:image\/[^"']+)["']/gi) + return blobs + } - for (const match of matches) { - push(dataUrlToBlob(match[1])) + // Inline `` in the clipboard's HTML — but only for a copy + // that carried no text of its own. A rich-text copy WITH prose is a text + // paste that happens to contain images, and its data URLs are the page's + // decorations rather than content: Discord ships a 32×5 blurhash placeholder + // beside every image embed, so copying a thread attached a blank thumbnail + // and (because an image paste swallows the event) dropped the text entirely. + if (!text) { + for (const match of clipboard.getData('text/html').matchAll(INLINE_IMAGE_SRC_RE)) { + const blob = dataUrlToBlob(match[1]) + + if (blob && blob.size >= MIN_INLINE_IMAGE_BYTES) { + push(blob) } } } @@ -90,7 +153,17 @@ export function extractClipboardImageBlobs(clipboard: DataTransfer): Blob[] { return blobs } -/** Caret-anchored text before the cursor, or null if the selection isn't a collapsed caret inside `editor`. */ +/** Caret-anchored text before the cursor, or null if the selection isn't a + * collapsed caret inside `editor`. + * + * Chips are ATOMIC to trigger detection: a committed pill must not leak its + * label text into the string the trigger regexes see. A `/work` pill whose + * label serialized into this text made the `^`-anchored command regex treat + * everything after it as that command's argument — which silenced the `@` + * popover for the rest of the message (`/work @Desk` → no trigger → the + * typed path never chips and submits as plain text). Each chip contributes + * an object-replacement placeholder instead, and
contributes a newline + * so a trigger at the start of a wrapped line still detects. */ export function textBeforeCaret(editor: HTMLDivElement): string | null { const sel = window.getSelection() const range = sel?.rangeCount ? sel.getRangeAt(0) : null @@ -99,24 +172,59 @@ export function textBeforeCaret(editor: HTMLDivElement): string | null { return null } - const before = range.cloneRange() - before.selectNodeContents(editor) - before.setEnd(range.startContainer, range.startOffset) + return serializeTextBefore(editor, range.startContainer, range.startOffset) +} + +/** How many characters of directive scope the caret is sitting inside (`@url:` + * with nothing typed after it), or 0. A paste lands INTO that scope: the scope + * text is consumed rather than left in front of the chip as leftover syntax. */ +export function openDirectiveScope(editor: HTMLDivElement): number { + const trigger = detectTrigger(textBeforeCaret(editor) ?? '') - return before.toString() + return trigger?.kind === '@' && trigger.scope && !trigger.value ? trigger.tokenLength : 0 } export function detectTrigger(textBefore: string): TriggerState | null { - const slash = SLASH_TRIGGER_RE.exec(textBefore) + // An inline `/skill` is a reference dropped into prose, so it carries no args + // and the whole match is the token the chip replaces. Checked before the + // anchored command shape so a second slash isn't mistaken for the first + // command's argument. + const inline = SLASH_INLINE_TRIGGER_RE.exec(textBefore) - if (slash) { - return { kind: '/', query: slash[2], tokenLength: 1 + slash[2].length } + if (inline) { + const query = inline[2] ?? '' + + return { inline: true, kind: '/', query, tokenLength: 1 + query.length, value: query } + } + + const command = SLASH_COMMAND_TRIGGER_RE.exec(textBefore) + + if (command) { + return { kind: '/', query: command[2], tokenLength: 1 + command[2].length, value: command[2] } } const at = AT_TRIGGER_RE.exec(textBefore) if (at) { - return { kind: '@', query: at[2], tokenLength: 1 + at[2].length } + const query = at[2] + const scoped = AT_SCOPE_RE.exec(query) + + return { + kind: '@', + query, + ...(scoped ? { scope: scoped[1] as DirectiveScope } : {}), + tokenLength: 1 + query.length, + value: scoped ? (scoped[2] ?? '') : query + } + } + + // After `@` so a directive starter's colon (`@file:`) stays an `@` query. + // Rides the reactions opt-in (Settings → Appearance) — both are one + // "emoji features" surface, off by default. + const emoji = $reactionsEnabled.get() ? EMOJI_TRIGGER_RE.exec(textBefore) : null + + if (emoji) { + return { kind: ':', query: emoji[2], tokenLength: 1 + emoji[2].length, value: emoji[2] } } return null diff --git a/apps/desktop/src/app/chat/composer/trigger-popover-parity.test.tsx b/apps/desktop/src/app/chat/composer/trigger-popover-parity.test.tsx new file mode 100644 index 000000000000..e3f2ab08eba4 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/trigger-popover-parity.test.tsx @@ -0,0 +1,151 @@ +import { render, screen } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' + +import { ComposerTriggerPopover } from './trigger-popover' + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + composer: { + lookupLoading: 'Loading…', + lookupNoMatches: 'No matches', + lookupTry: 'Try', + lookupOr: 'or' + } + } + }) +})) + +function atItem(type: string, display: string, rawText: string, meta = '') { + return { + id: `${rawText}|0`, + type, + label: display, + metadata: { icon: type, display, meta, rawText, insertId: display } + } +} + +function slashItem(command: string, group: string, meta = '') { + return { + id: `${command}|0`, + type: 'slash', + label: command.slice(1), + metadata: { command, display: command, meta, group, action: '', rawText: command } + } +} + +const noop = () => {} + +/** The rendered shape of one row: does it have an icon, and how is it laid out? + * Icons are codicons — an ``, not an SVG. */ +function rowShape(root: HTMLElement) { + const row = root.querySelector('button') as HTMLElement + const icon = row.querySelector('i.codicon') + + return { + hasIcon: Boolean(icon), + iconName: icon?.className.match(/codicon-([\w-]+)/)?.[1], + classes: row.className + } +} + +describe('@ and / are one menu', () => { + it('a slash row has an icon, just like an @ row', () => { + const at = render( + + ) + + const atShape = rowShape(at.container) + at.unmount() + + const slash = render( + + ) + + const slashShape = rowShape(slash.container) + + // The whole point: `/` used to render a stacked, icon-less row. + expect(slashShape.hasIcon).toBe(true) + expect(atShape.hasIcon).toBe(true) + expect(slashShape.classes).toBe(atShape.classes) + + // And the glyph reflects the kind, not one generic bullet. + expect(slashShape.iconName).toBe('zap') + expect(atShape.iconName).toBe('folder') + }) + + it('renders the name and description for both kinds', () => { + const { rerender } = render( + + ) + + expect(screen.getByText('/work')).toBeTruthy() + expect(screen.getByText('Start in a worktree')).toBeTruthy() + + rerender( + + ) + + expect(screen.getByText('src/main.tsx')).toBeTruthy() + expect(screen.getByText('src')).toBeTruthy() + }) + + it('an emoji row stays icon-less — the emoji IS the icon', () => { + const { container } = render( + + ) + + expect(rowShape(container).hasIcon).toBe(false) + }) + + it('labels the active browse scope from the shared vocabulary', () => { + render( + + ) + + expect(screen.getByText('Folders')).toBeTruthy() + }) +}) diff --git a/apps/desktop/src/app/chat/composer/trigger-popover.tsx b/apps/desktop/src/app/chat/composer/trigger-popover.tsx index da52f1dd088c..32b403230340 100644 --- a/apps/desktop/src/app/chat/composer/trigger-popover.tsx +++ b/apps/desktop/src/app/chat/composer/trigger-popover.tsx @@ -1,62 +1,82 @@ import type { Unstable_TriggerItem } from '@assistant-ui/core' import { Fragment } from 'react' +import { referenceKind, referenceStyle } from '@/components/assistant-ui/reference-kinds' import { Codicon } from '@/components/ui/codicon' import { GlyphSpinner } from '@/components/ui/glyph-spinner' import { useI18n } from '@/i18n' import { cn } from '@/lib/utils' import { COMPLETION_DRAWER_BELOW_CLASS, COMPLETION_DRAWER_CLASS, CompletionDrawerEmpty } from './completion-drawer' +import type { DirectiveScope } from './text-utils' -const AT_ICON_BY_TYPE: Record = { - diff: 'diff', - file: 'book', - folder: 'folder', - git: 'git-branch', - image: 'file-media', - simple: 'symbol-misc', - staged: 'diff-added', - tool: 'tools', - url: 'globe' +interface RowMeta { + display?: string + group?: string + meta?: string } -function atIcon(item: Unstable_TriggerItem) { - const meta = item.metadata as { rawText?: string } | undefined +/** The kind a row represents, for its icon. `@` rows carry it as the item type; + * `/` rows carry it as the completion group (Skills / Themes / Commands). */ +function rowKind(item: Unstable_TriggerItem, isSlash: boolean): string { + const meta = item.metadata as (RowMeta & { rawText?: string }) | undefined + + if (isSlash) { + const group = meta?.group?.trim() + + return group === 'Skills' ? 'skill' : group === 'Themes' ? 'theme' : 'command' + } + + // The gateway's simple refs (`@diff`, `@staged`) share one item type, so the + // glyph comes from the directive itself. const raw = meta?.rawText || item.label if (raw.startsWith('@diff')) { - return AT_ICON_BY_TYPE.diff + return 'diff' } if (raw.startsWith('@staged')) { - return AT_ICON_BY_TYPE.staged + return 'staged' } - return AT_ICON_BY_TYPE[item.type] || AT_ICON_BY_TYPE.simple + return item.type } -interface RowMeta { - display?: string - group?: string - meta?: string -} - -const ROW_BASE_CLASS = [ - 'relative flex w-full cursor-default select-none rounded-md px-2 py-1 text-left', +const ROW_CLASS = [ + 'relative flex w-full cursor-default select-none items-center gap-2 rounded-md px-2 py-1 text-left', 'outline-hidden transition-colors hover:bg-(--ui-bg-tertiary)', 'data-[highlighted]:bg-(--ui-bg-tertiary) data-[highlighted]:text-foreground' ].join(' ') +const GROUP_HEADER_CLASS = + 'select-none px-2 pb-0.5 text-[0.625rem] font-semibold uppercase tracking-wider text-(--ui-text-tertiary)' + interface ComposerTriggerPopoverProps { activeIndex: number items: readonly Unstable_TriggerItem[] - kind: '@' | '/' + kind: '@' | '/' | ':' loading: boolean onHover: (index: number) => void onPick: (item: Unstable_TriggerItem) => void placement?: 'bottom' | 'top' + /** The `@kind:` browse the list is filtered to, when there is one. Rendered + * as a header so the scope reads as the mode it is — the raw `@folder:` in + * the editor otherwise looks like syntax the user has to finish by hand. */ + scope?: DirectiveScope } +/** + * The composer's completion list, for every trigger. + * + * `@` and `/` render through the SAME row: icon, name, description. They used + * to be two layouts in one file — `@` horizontal with an icon, `/` stacked with + * none — which is why picking a file and picking a skill felt like features + * from different apps. Icons and accents come from the shared reference + * vocabulary, so a row looks like the chip it will become. + * + * `:` emoji is the one exception: the emoji IS the icon, so it renders as a + * single display string (Slack's exact shape). + */ export function ComposerTriggerPopover({ activeIndex, items, @@ -64,11 +84,13 @@ export function ComposerTriggerPopover({ loading, onHover, onPick, - placement = 'top' + placement = 'top', + scope }: ComposerTriggerPopoverProps) { const { t } = useI18n() const copy = t.composer const isSlash = kind === '/' + const isEmoji = kind === ':' let lastGroup: string | undefined @@ -80,6 +102,7 @@ export function ComposerTriggerPopover({ onMouseDown={event => event.preventDefault()} role="listbox" > + {scope &&
{referenceStyle(scope).label}
} {items.length === 0 ? ( loading ? (
@@ -93,6 +116,10 @@ export function ComposerTriggerPopover({ {copy.lookupTry} @file: {copy.lookupOr}{' '} @folder:. + ) : isEmoji ? ( + <> + {copy.lookupTry} :joy:. + ) : ( <> {copy.lookupTry} /help. @@ -110,58 +137,28 @@ export function ComposerTriggerPopover({ const isFirstHeader = lastGroup === undefined lastGroup = group || lastGroup const active = index === activeIndex + const refKind = referenceKind(rowKind(item, isSlash)) return ( - {showHeader && ( -
- {group} -
- )} + {showHeader &&
{group}
} + + + + ) + } + + render() + + expect(screen.getByTestId('thread')).toBeTruthy() + expect(threadRenderCount.current).toBe(1) + + fireEvent.click(screen.getByRole('button', { name: /parent tick/i })) + + // memo(ChatView) with stable props must absorb the parent's idle tick — + // the transcript (Thread) must not re-render. This is PR #38470's contract. + expect(threadRenderCount.current).toBe(1) + }) +}) diff --git a/apps/desktop/src/app/chat/index.tsx b/apps/desktop/src/app/chat/index.tsx index bd18be0c4834..7a0c0a2674c3 100644 --- a/apps/desktop/src/app/chat/index.tsx +++ b/apps/desktop/src/app/chat/index.tsx @@ -1,14 +1,16 @@ import { type AppendMessage, AssistantRuntimeProvider, type ThreadMessage } from '@assistant-ui/react' import { useStore } from '@nanostores/react' import { useQuery } from '@tanstack/react-query' +import type { ReadableAtom } from 'nanostores' import type * as React from 'react' -import { Suspense, useCallback, useEffect, useMemo } from 'react' -import { useLocation } from 'react-router-dom' +import { memo, Suspense, useCallback, useEffect, useMemo, useState } from 'react' +import { useLocation } from 'react-router' import type { SubmitTextOptions } from '@/app/session/hooks/use-prompt-actions/utils' import { Thread } from '@/components/assistant-ui/thread' import { Backdrop } from '@/components/Backdrop' import { COMPOSER_HEART_CONFIG, HeartField } from '@/components/chat/vibe-hearts' +import { usePaneVisible } from '@/components/pane-shell/pane-visibility' import { $sessionTileDragging, $sessionTileEdgeHover } from '@/components/pane-shell/tree/store' import { PromptOverlays } from '@/components/prompt-overlays' import { Button } from '@/components/ui/button' @@ -37,12 +39,13 @@ import { $sessions, resolveComposerSessionKey, sessionMatchesStoredId, - sessionPinId + sessionPinId, + shouldMigrateComposerScope } from '@/store/session' import { isSecondaryWindow, isWatchWindow } from '@/store/windows' import type { ModelOptionsResponse } from '@/types/hermes' -import { routeSessionId } from '../routes' +import { primaryRouteSelectedSessionId, routeSessionId } from '../routes' import { titlebarHeaderBaseClass, titlebarHeaderShadowClass, titlebarHeaderTitleClass } from '../shell/titlebar' import { ChatDropOverlay } from './chat-drop-overlay' @@ -69,7 +72,7 @@ interface ChatViewProps extends Omit, 'onSubmit'> { onCancel: () => Promise | void onAddContextRef: (refText: string, label?: string, detail?: string) => void onAddUrl: (url: string) => void - onBranchInNewChat: (messageId: string) => void + onBranchInNewChat?: (messageId: string) => void maxVoiceRecordingSeconds?: number onAttachImageBlob: (blob: Blob) => Promise | boolean | void onAttachDroppedItems: (candidates: DroppedFile[]) => Promise | boolean | void @@ -174,6 +177,30 @@ interface ChatRuntimeBoundaryProps { const NO_MESSAGES: ChatMessage[] = [] +/** + * The view's $messages, live only while this surface is the VISIBLE tab. + * + * Keep-alive keeps every ever-active tab MOUNTED (tree-group.tsx), so without + * this gate a hidden tab re-renders its entire thread on every streaming + * delta flush (~30×/s) — five busy tabs quintuple the per-token render cost + * and the app crawls. Hidden tabs freeze their transcript instead (status + * dots stay live through the separate status atoms) and catch up in one + * commit on reveal — the subscribe fires immediately with the current value. + */ +function useMessagesWhileVisible($messages: ReadableAtom): ChatMessage[] { + const visible = usePaneVisible() + const [messages, setMessages] = useState(() => $messages.get()) + + // nanostores types the listener value ReadonlyIfObject; the store publishes + // a fresh array per flush, so the cast is safe and avoids a per-token clone. + useEffect( + () => (visible ? $messages.subscribe(value => setMessages(value as ChatMessage[])) : undefined), + [$messages, visible] + ) + + return messages +} + /** * Owns the $messages subscription and the assistant-ui external-store runtime. * @@ -193,7 +220,7 @@ function ChatRuntimeBoundary({ onThreadMessagesChange, suppressMessages }: ChatRuntimeBoundaryProps) { - const storeMessages = useStore(useSessionView().$messages) + const storeMessages = useMessagesWhileVisible(useSessionView().$messages) const messages = suppressMessages ? NO_MESSAGES : storeMessages const runtimeMessageRepository = useRuntimeMessageRepository(messages) @@ -213,7 +240,10 @@ function ChatRuntimeBoundary({ return {children} } -export function ChatView({ +// Memoized: the tile caller (session-tile.tsx) and the contrib surface re-render +// on idle ticks unrelated to the chat; with stable callback props (hoisted to +// useCallback at the call sites) memo() lets the whole chat shell skip those. +export const ChatView = memo(function ChatView({ className, gateway, modelMenuContent, @@ -285,22 +315,34 @@ export function ChatView({ const resumeExhaustedSessionId = useStore($resumeExhaustedSessionId) // Durable composer/queue scope (lineage root) so auto-compression tip rotation - // does not wipe an in-progress draft or orphan /queue entries. - const queueSessionKey = useMemo( - () => resolveComposerSessionKey(selectedSessionId, sessions), - [selectedSessionId, sessions] - ) + // does not wipe an in-progress draft or orphan /queue entries. For the + // primary view, the route is authoritative over the store selection — the + // latter can be momentarily null/stale mid-switch, which used to leak into + // the composer's scope key (#59305). A tile has no route, so it always uses + // its own selection directly. + const queueSessionKey = useMemo(() => { + const effectiveSelectedSessionId = isPrimary + ? primaryRouteSelectedSessionId(location.pathname, selectedSessionId) + : selectedSessionId + + return resolveComposerSessionKey(effectiveSelectedSessionId, sessions) + }, [isPrimary, location.pathname, selectedSessionId, sessions]) // When the tip row arrives after compression, migrate any tip-keyed stash onto // the durable lineage key before the composer remounts onto that key. + // + // ONLY same-conversation rekeys (tip → root). The route-driven queueSessionKey + // can flip to Session B a frame before the store selection leaves Session A; + // migrating on bare inequality would re-home A's queued prompts onto B and + // auto-drain them into the wrong chat. useEffect(() => { - if (!selectedSessionId || !queueSessionKey || selectedSessionId === queueSessionKey) { + if (!shouldMigrateComposerScope(selectedSessionId, queueSessionKey, sessions)) { return } migrateSessionDraft(selectedSessionId, queueSessionKey) migrateQueuedPrompts(selectedSessionId, queueSessionKey) - }, [queueSessionKey, selectedSessionId]) + }, [queueSessionKey, selectedSessionId, sessions]) // Transcript-side stops (the streaming message's hover Stop, the runtime's // cancel) are explicit halts, same as the composer's Stop button: park any @@ -437,6 +479,7 @@ export function ChatView({ 'relative isolate flex h-full min-w-0 flex-col overflow-hidden bg-(--ui-chat-surface-background)', className )} + data-chat-surface="" data-composer-target={composerScope.target} data-session-anchor={sessionAnchor} > @@ -508,7 +551,7 @@ export function ChatView({ config={COMPOSER_HEART_CONFIG} style={{ top: 0, - bottom: 'calc(var(--composer-measured-height) + var(--status-stack-measured-height) + 0.25rem)' + bottom: 'calc(var(--composer-measured-height) + 0.25rem)' }} /> )} @@ -556,4 +599,4 @@ export function ChatView({
) -} +}) diff --git a/apps/desktop/src/app/chat/perf-probe.tsx b/apps/desktop/src/app/chat/perf-probe.tsx index 987d89fb428d..475925823875 100644 --- a/apps/desktop/src/app/chat/perf-probe.tsx +++ b/apps/desktop/src/app/chat/perf-probe.tsx @@ -1,7 +1,18 @@ import { Profiler, type ProfilerOnRenderCallback, type ReactNode } from 'react' +import { $terminalTakeover, setTerminalTakeover } from '@/app/right-sidebar/store' +import { writeAgentTerminalChunk } from '@/app/right-sidebar/terminal/agent-terminal-stream' +import { + $activeTerminalId, + $terminals, + createTerminal, + ensureAgentTerminal, + selectTerminal, + type TerminalEntry +} from '@/app/right-sidebar/terminal/terminals' +import { $repoStatusByCwd } from '@/store/coding-status' import { $gateway } from '@/store/gateway' -import { $messages, setBusy, setMessages } from '@/store/session' +import { $currentCwd, $messages, setBusy, setCurrentCwdTransient, setMessages } from '@/store/session' type Sample = { id: string @@ -38,6 +49,12 @@ declare global { * backend) doesn't contaminate frame-pacing numbers. */ connected: () => boolean + /** Mount files + multiple xterms for the synthetic right-pane scenario. */ + rightPaneSetup: (opts: { cwd: string; terminals?: number }) => { procId: string; terminalIds: string[] } + rightPaneGit: (path: string, kind?: 'added' | 'conflicted' | 'modified') => void + rightPaneReset: () => void + rightPaneSelect: (id: string) => void + rightPaneWrite: (procId: string, chunk: string) => void reset: () => void snapshotMsgs: () => number } @@ -102,11 +119,34 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) { let baseline: ReturnType | null = null let activeHandle: SyntheticDriverHandle | null = null + let rightPaneBaseline: + | null + | { + activeTerminalId: null | string + cwd: string + repoStatusByCwd: ReturnType + takeover: boolean + terminals: readonly TerminalEntry[] + } = null + const stop = () => { activeHandle = null setBusy(false) } + const resetRightPane = () => { + if (!rightPaneBaseline) { + return + } + + setTerminalTakeover(rightPaneBaseline.takeover) + $terminals.set(rightPaneBaseline.terminals) + $activeTerminalId.set(rightPaneBaseline.activeTerminalId) + $repoStatusByCwd.set(rightPaneBaseline.repoStatusByCwd) + setCurrentCwdTransient(rightPaneBaseline.cwd) + rightPaneBaseline = null + } + // One synthetic turn's worth of mixed markdown — prose, a list, a fenced // code block, inline code, a link, and a short table — so a loaded transcript // exercises the same render cost (Streamdown blocks, code cards) a real one @@ -166,6 +206,69 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) { return false } }, + rightPaneGit: (path, kind = 'modified') => { + const file = { + conflicted: kind === 'conflicted', + path, + staged: false, + unstaged: kind === 'modified', + untracked: kind === 'added' + } + + const cwd = $currentCwd.get().trim() + $repoStatusByCwd.set({ + ...$repoStatusByCwd.get(), + [cwd]: { + added: 0, + ahead: 0, + behind: 0, + branch: 'perf', + changed: 1, + conflicted: kind === 'conflicted' ? 1 : 0, + defaultBranch: 'main', + detached: false, + files: [file], + removed: 0, + staged: 0, + unstaged: kind === 'modified' ? 1 : 0, + untracked: kind === 'added' ? 1 : 0 + } + }) + }, + rightPaneReset: resetRightPane, + rightPaneSelect: selectTerminal, + rightPaneSetup: ({ cwd, terminals = 3 }) => { + resetRightPane() + rightPaneBaseline = { + activeTerminalId: $activeTerminalId.get(), + cwd: $currentCwd.get(), + repoStatusByCwd: $repoStatusByCwd.get(), + takeover: $terminalTakeover.get(), + terminals: $terminals.get() + } + + setCurrentCwdTransient(cwd) + const terminalIds = [createTerminal(cwd)] + let procId = '' + + for (let index = 1; index < Math.max(1, terminals); index += 1) { + procId = `right-pane-perf-${Date.now()}-${index}` + const id = ensureAgentTerminal(procId, `perf output ${index}`) + + if (id) { + terminalIds.push(id) + } + } + + if (procId) { + selectTerminal(terminalIds.at(-1) ?? terminalIds[0]) + } + + setTerminalTakeover(true) + + return { procId, terminalIds } + }, + rightPaneWrite: (procId, chunk) => writeAgentTerminalChunk(procId, chunk), loadTranscript: (turns = 200) => { if (!baseline) { baseline = $messages.get() @@ -190,6 +293,7 @@ if (typeof window !== 'undefined' && !window.__PERF_DRIVE__) { }, reset: () => { activeHandle?.stop() + resetRightPane() if (baseline) { setMessages(baseline) diff --git a/apps/desktop/src/app/chat/right-rail/preview-artifact.test.tsx b/apps/desktop/src/app/chat/right-rail/preview-artifact.test.tsx new file mode 100644 index 000000000000..5b1a84e20988 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-artifact.test.tsx @@ -0,0 +1,112 @@ +import { act, cleanup, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it } from 'vitest' + +import { $artifactRegistry, $artifactVersionSelection, artifactPreviewTarget, upsertArtifact } from '@/store/artifacts' + +import { ArtifactPreview } from './preview-artifact' + +function register(title: string, kind: 'code' | 'html' | 'svg', content: string) { + const result = upsertArtifact('session-1', { kind, language: kind === 'code' ? 'python' : kind, title }, content) + + if (!result) { + throw new Error('artifact did not register') + } + + return result +} + +async function renderArtifact(artifactId: string) { + const record = $artifactRegistry.get()['session-1']!.find(item => item.id === artifactId)! + + await act(async () => { + render() + }) +} + +describe('ArtifactPreview', () => { + afterEach(() => { + cleanup() + $artifactRegistry.set({}) + $artifactVersionSelection.set({}) + }) + + it('renders html in a scripts-only sandboxed frame the parent app is unreachable from', async () => { + const { artifactId } = register('Dashboard', 'html', '

Hi

') + await renderArtifact(artifactId) + + const frame = screen.getByTitle('Dashboard') as HTMLIFrameElement + + expect(frame.getAttribute('sandbox')).toBe('allow-scripts') + expect(frame.srcdoc).toContain('

Hi

') + // No allow-same-origin: scripts inside cannot reach the renderer's origin. + expect(frame.getAttribute('sandbox')).not.toContain('same-origin') + }) + + it('strips scripts out of svg before it renders inline', async () => { + const { artifactId } = register( + 'Logo', + 'svg', + '' + ) + + await renderArtifact(artifactId) + + expect(document.querySelector('svg')).not.toBeNull() + expect(document.querySelector('svg script')).toBeNull() + }) + + it('offers only the source view for code, which has nothing to render', async () => { + const { artifactId } = register('Solver', 'code', 'print("hi")') + await renderArtifact(artifactId) + + expect(screen.queryByRole('button', { name: /rendered/i })).toBeNull() + }) + + it('shows the version stepper once an artifact has history, and follows the selection', async () => { + register('Dashboard', 'html', '

v1

') + const { artifactId } = register('Dashboard', 'html', '

v2

') + await renderArtifact(artifactId) + + expect(screen.getByText('v2 of 2')).toBeTruthy() + expect((screen.getByTitle('Dashboard') as HTMLIFrameElement).srcdoc).toContain('v2') + + await act(async () => { + $artifactVersionSelection.set({ [artifactId]: 0 }) + }) + + expect(screen.getByText('v1 of 2')).toBeTruthy() + expect((screen.getByTitle('Dashboard') as HTMLIFrameElement).srcdoc).toContain('v1') + }) + + it('hides the stepper for a single-version artifact', async () => { + const { artifactId } = register('Dashboard', 'html', '

only

') + await renderArtifact(artifactId) + + expect(screen.queryByText('v1 of 1')).toBeNull() + }) + + it('picks up a new version in an already-open tab', async () => { + const { artifactId } = register('Dashboard', 'html', '

v1

') + await renderArtifact(artifactId) + + await act(async () => { + register('Dashboard', 'html', '

v2

') + }) + + expect((screen.getByTitle('Dashboard') as HTMLIFrameElement).srcdoc).toContain('v2') + }) + + it('falls back to an empty state when the registry no longer has the artifact', async () => { + const { artifactId } = register('Dashboard', 'html', '

gone

') + const record = $artifactRegistry.get()['session-1']!.find(item => item.id === artifactId)! + const target = artifactPreviewTarget(record) + + $artifactRegistry.set({}) + + await act(async () => { + render() + }) + + expect(screen.queryByTitle('Dashboard')).toBeNull() + }) +}) diff --git a/apps/desktop/src/app/chat/right-rail/preview-artifact.tsx b/apps/desktop/src/app/chat/right-rail/preview-artifact.tsx new file mode 100644 index 000000000000..07d60393a8e4 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-artifact.tsx @@ -0,0 +1,263 @@ +import { useStore } from '@nanostores/react' +import DOMPurify from 'dompurify' +import { useEffect, useMemo, useState } from 'react' + +import { CopyButton } from '@/components/ui/copy-button' +import { Tip } from '@/components/ui/tooltip' +import { useI18n } from '@/i18n' +import { artifactDownloadName, type ArtifactKind } from '@/lib/artifact-detect' +import { downloadTextFile } from '@/lib/download-text' +import { ChevronLeft, ChevronRight, Download, ExternalLink } from '@/lib/icons' +import { $artifactRegistry, $artifactVersionSelection, findArtifact, selectArtifactVersion } from '@/store/artifacts' +import { notifyError } from '@/store/notifications' +import type { PreviewTarget } from '@/store/preview' + +import { PreviewEmptyState, PreviewModeSwitcher, type PreviewViewMode, SourceView } from './preview-file' + +const MIME_BY_KIND = { code: 'text/plain', html: 'text/html', svg: 'image/svg+xml' } as const + +// Shiki has no `svg` grammar; code artifacts keep their detected fence language. +const SOURCE_LANGUAGE_BY_KIND: Record = { + code: undefined, + html: 'html', + svg: 'xml' +} + +const HEADER_BUTTON_CLASS = + 'flex items-center gap-1 rounded-md px-1.5 text-[0.625rem] font-bold text-muted-foreground transition-colors hover:bg-accent hover:text-foreground disabled:pointer-events-none disabled:opacity-40' + +/** Wrap an HTML fragment in a minimal document shell; full documents pass + * through untouched. Keeps generated fragments (no /) rendering + * with sane defaults instead of quirks-mode soup. */ +function composeArtifactHtml(content: string): string { + if (/]|', + '', + '', + content, + '' + ].join('\n') +} + +/** Write the composed document to a real temp file through the existing + * buffer-save IPC, then hand it to the OS browser. A blob/data URL can't + * cross into the OS default browser, so a file on disk is the honest path. */ +async function openHtmlInBrowser(content: string): Promise { + const bridge = window.hermesDesktop + + if (!bridge?.saveImageBuffer || !bridge.openExternal) { + throw new Error('Desktop bridge unavailable') + } + + const bytes = new TextEncoder().encode(composeArtifactHtml(content)) + const path = await bridge.saveImageBuffer(bytes, '.html') + + if (!path) { + throw new Error('Could not write artifact file') + } + + const fileUrl = `file://${path.startsWith('/') ? '' : '/'}${path.replace(/\\/g, '/')}` + + if (bridge.openPreviewInBrowser) { + await bridge.openPreviewInBrowser(fileUrl) + + return + } + + await bridge.openExternal(fileUrl) +} + +/** + * Live view for renderable artifact content. + * + * HTML runs in an `