diff --git a/.audit-tool-lock.json b/.audit-tool-lock.json new file mode 100644 index 0000000000000..b66d32afbacc2 --- /dev/null +++ b/.audit-tool-lock.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "tools": { + "zizmor": { + "version": "1.30.0", + "sha256": "sha256:98c426d9668ba03d7444acdb8a5f0eb44ba187ddffeba15833b605a384d50879" + }, + "import-linter": { + "version": "2.14", + "sha256": "sha256:a6558ea7f4f0fea70cf21f63eac9d37018882f8ed529bb3313ba28e1a2bf279e", + "distribution_sha256": "sha256:de0858c9dc7eeca513b6daa31dec7099939addfb1f089536ec806a0b1611bddb" + }, + "pip-audit": { + "version": "2.10.1", + "sha256": "sha256:ade9c86ba46074ca0224526121f061c67fadafabf00294c7d52d6436221604de", + "distribution_sha256": "sha256:5691c79b88f4c5c5b9f8f7642c9708d12275998fe7e523144b5c132807ac135c" + } + }, + "uv": { + "version": "0.12.6", + "sha256": "sha256:e8929237934c8679686428f5a7736c7ae7a5fe7a33b0504d1b03446cdbc43c94" + } +} diff --git a/.env.example b/.env.example index 5380b988de136..2e1cbb849a194 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,7 @@ # Default model is configured in ~/.hermes/config.yaml (model.default). # Use 'hermes model' or 'hermes setup' to change it. # LLM_MODEL is no longer read from .env — this line is kept for reference only. -# LLM_MODEL=anthropic/claude-opus-4.6 +# LLM_MODEL=openai-codex/gpt-5.5 # ============================================================================= # LLM PROVIDER (NovitaAI) diff --git a/.github/actions/get-app-token/action.yml b/.github/actions/get-app-token/action.yml index 2aaf303ab2de7..84bd1c877af22 100644 --- a/.github/actions/get-app-token/action.yml +++ b/.github/actions/get-app-token/action.yml @@ -12,7 +12,8 @@ description: >- Composite actions cannot access contexts directly, so callers pass the public vars.APP_CLIENT_ID and protected secrets.APP_PRIVATE_KEY as inputs. - When the private key is empty, the fallback fires. + When either credential is empty, the fallback fires rather than attempting + to mint with a partial credential pair. inputs: client-id: @@ -31,6 +32,30 @@ inputs: description: Comma- or newline-separated repositories to scope within the installation owner. required: false default: '' + permission-actions: + description: Actions API permission requested for the token (read or write). + required: false + default: '' + permission-checks: + description: Checks API permission requested for the token (read or write). + required: false + default: '' + permission-contents: + description: Repository contents permission requested for the token (read or write). + required: false + default: '' + permission-issues: + description: Issues permission requested for the token (read or write). + required: false + default: '' + permission-pull-requests: + description: Pull request permission requested for the token (read or write). + required: false + default: '' + permission-statuses: + description: Commit status permission requested for the token (read or write). + required: false + default: '' outputs: token: @@ -45,8 +70,9 @@ runs: shell: bash env: CLIENT_ID: ${{ inputs.client-id }} + PRIVATE_KEY: ${{ inputs.private-key }} run: | - if [ -n "$CLIENT_ID" ]; then + if [ -n "$CLIENT_ID" ] && [ -n "$PRIVATE_KEY" ]; then echo "has_app=true" >> "$GITHUB_OUTPUT" else echo "has_app=false" >> "$GITHUB_OUTPUT" @@ -61,6 +87,12 @@ runs: private-key: ${{ inputs.private-key }} owner: ${{ inputs.owner }} repositories: ${{ inputs.repositories }} + permission-actions: ${{ inputs.permission-actions }} + permission-checks: ${{ inputs.permission-checks }} + permission-contents: ${{ inputs.permission-contents }} + permission-issues: ${{ inputs.permission-issues }} + permission-pull-requests: ${{ inputs.permission-pull-requests }} + permission-statuses: ${{ inputs.permission-statuses }} - name: Fall back to GITHUB_TOKEN id: fallback diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c64dc70b2c3de..4c22de37597eb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,12 +18,10 @@ on: pull_request: push: branches: [main] + workflow_dispatch: permissions: contents: read - pull-requests: write # needed by lint (PR comment) + supply-chain review_status - actions: read # needed by osv-scanner (SARIF upload) - security-events: write # needed by osv-scanner (SARIF upload) concurrency: group: ci-${{ github.ref }} @@ -197,17 +195,24 @@ jobs: supply-chain: name: Supply-chain scan needs: detect - if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true') + if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true' || needs.detect.outputs.npm_lock == 'true') + permissions: + contents: read + pull-requests: read uses: ./.github/workflows/supply-chain-audit.yml with: event_name: ${{ needs.detect.outputs.event_name }} scan: ${{ needs.detect.outputs.scan == 'true' }} deps: ${{ needs.detect.outputs.deps == 'true' }} + npm_lock: ${{ needs.detect.outputs.npm_lock == 'true' }} review-labels: name: Review label gate needs: [detect, supply-chain] if: always() && needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.ci_review == 'true' || needs.detect.outputs.mcp_catalog == 'true' || needs.supply-chain.outputs.critical_findings == 'true') + permissions: + contents: read + pull-requests: read uses: ./.github/workflows/review-labels.yml with: ci_review: ${{ needs.detect.outputs.ci_review == 'true' }} @@ -217,6 +222,10 @@ jobs: osv-scanner: name: OSV scan + permissions: + actions: read + contents: read + security-events: write # PR scans use the artifact/review-status path below. Direct main pushes # are scanned by osv-scanner.yml's push trigger so they can publish SARIF. if: github.event_name == 'pull_request' diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 26e11ee9fb3c6..d563b8588602e 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -24,9 +24,6 @@ on: permissions: contents: read - actions: read - pages: write - id-token: write concurrency: group: pages @@ -48,6 +45,11 @@ jobs: deploy-docs: if: github.repository == 'NousResearch/hermes-agent' + permissions: + actions: read + contents: read + id-token: write + pages: write runs-on: ubuntu-latest timeout-minutes: 30 environment: @@ -62,6 +64,8 @@ jobs: with: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-actions: read + permission-contents: read - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7e6b916f2ae96..bc2cea38c1420 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,8 @@ name: Docker Build, Test, and Publish on: + push: + branches: [main] # This workflow owns its own triggers. ci.yml does not call it. # A reusable-workflow call eeps the caller run in progress for that full time. # GitHub refuses ``gh run rerun`` on a run that is still in progress. @@ -10,10 +12,9 @@ on: # Trusted main pushes resolve the environment-scoped Docker Hub secrets in # this same workflow, never across a workflow boundary. pull_request: - push: - branches: [main] release: types: [published] + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml index 8692f05d7a413..daeac7838514b 100644 --- a/.github/workflows/e2e-desktop.yml +++ b/.github/workflows/e2e-desktop.yml @@ -107,8 +107,15 @@ jobs: # is always fresh — no separate build step needed. - name: Run Playwright E2E tests working-directory: apps/desktop + env: + GITHUB_REF_NAME_SAFE: ${{ github.ref_name }} + CI: 'true' + # Ensure no real API keys leak into the test env. + OPENROUTER_API_KEY: '' + OPENAI_API_KEY: '' + NOUS_API_KEY: '' run: | - if [ "${{ github.ref_name }}" = "main" ]; then + if [ "$GITHUB_REF_NAME_SAFE" = "main" ]; then echo "On main — generating/updating baseline screenshots" npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ npx playwright test --reporter=list --update-snapshots @@ -117,12 +124,6 @@ jobs: npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ npx playwright test --reporter=list fi - env: - CI: 'true' - # Ensure no real API keys leak into the test env. - OPENROUTER_API_KEY: '' - OPENAI_API_KEY: '' - NOUS_API_KEY: '' # ── Save updated baselines to cache (main only) ─────────────────── - name: Save updated baselines to cache diff --git a/.github/workflows/js-autofix.yml b/.github/workflows/js-autofix.yml index bd4ee6d1c92dc..53e3b80f7eaea 100644 --- a/.github/workflows/js-autofix.yml +++ b/.github/workflows/js-autofix.yml @@ -35,16 +35,6 @@ name: auto-fix lint issues & formatting # on the current state. on: - push: - branches: [main] - paths: - - '**/*.js' - - '**/*.cjs' - - '**/*.mjs' - - '**/*.ts' - - '**/*.tsx' - - 'package.json' - - 'package-lock.json' workflow_dispatch: permissions: @@ -141,8 +131,10 @@ jobs: timeout-minutes: 15 environment: trusted-automation permissions: + checks: read # gh pr checks reads CheckRun entries contents: write # needed to push to bot/js-autofix pull-requests: write # needed for PR creation + auto-merge + statuses: read # gh pr checks also reads commit StatusContext entries steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -152,6 +144,10 @@ jobs: with: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-checks: read + permission-contents: write + permission-pull-requests: write + permission-statuses: read - name: Download patch uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6b4023ebe5060..741d86fa46273 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -53,12 +53,15 @@ jobs: - name: Determine base ref id: base + env: + BASE_BRANCH: ${{ github.base_ref }} + EVENT_NAME: ${{ inputs.event_name }} run: | # For PRs, diff against the merge base with the target branch. # For pushes to main, diff against the previous commit on main. - if [ "${{ inputs.event_name }}" = "pull_request" ]; then - BASE_SHA=$(git merge-base "origin/${{ github.base_ref }}" HEAD) - BASE_REF="origin/${{ github.base_ref }}" + if [ "$EVENT_NAME" = "pull_request" ]; then + BASE_SHA=$(git merge-base "origin/$BASE_BRANCH" HEAD) + BASE_REF="origin/$BASE_BRANCH" else BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || git rev-parse HEAD) BASE_REF="HEAD~1" diff --git a/.github/workflows/lockfile-diff.yml b/.github/workflows/lockfile-diff.yml index aa72e3190459e..d622771b08cf3 100644 --- a/.github/workflows/lockfile-diff.yml +++ b/.github/workflows/lockfile-diff.yml @@ -47,12 +47,14 @@ jobs: - name: Generate semantic lockfile diff id: diff + env: + BASE_BRANCH: ${{ github.base_ref }} run: | set -euo pipefail # Three-dot semantics by hand: diff from the merge base with the # target branch to the PR head, so changes that landed on main # after the branch point don't show up as this PR's doing. - BASE_SHA=$(git merge-base "origin/${{ github.base_ref }}" HEAD) + BASE_SHA=$(git merge-base "origin/$BASE_BRANCH" HEAD) echo "Merge base: ${BASE_SHA}" python3 scripts/ci/lockfile_diff.py \ --base "$BASE_SHA" \ diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 6f8d9b7299c74..ac87e18b594d8 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -13,7 +13,9 @@ name: Nix flake check on: pull_request: push: - branches: [main] + branches: + - main + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml index 173d361cc1396..d61298df7890f 100644 --- a/.github/workflows/osv-scanner.yml +++ b/.github/workflows/osv-scanner.yml @@ -14,16 +14,15 @@ name: OSV-Scanner # code patterns in PR diffs) by covering the orthogonal "currently-pinned # dep became known-vulnerable" case. # -# Uses Google's officially-recommended reusable workflow, pinned by SHA. -# Findings land in the repo's Security tab for direct main/scheduled scans. PR -# scans keep the artifact and unified review status without creating a -# baseline-mismatch check. -# fail-on-vuln is disabled so the job does not block merges on pre-existing -# vulnerabilities in pinned deps that we may need to patch deliberately. +# Uses Google's scanner and reporter actions pinned by SHA. The upstream +# reusable workflow wraps the same actions, but currently calls an unpinned +# download-artifact action internally; spelling the steps out here keeps this +# repo's full-SHA action policy intact. # -# The reusable workflow can't emit custom outputs, so a wrapper job -# downloads the SARIF result and summarizes the vulnerability count into -# a review_status for the unified PR comment. +# Findings land in the repo's Security tab (Code Scanning > OSV-Scanner). +# fail-on-vuln is disabled so the job does not block merges on pre-existing +# vulnerabilities in pinned deps that we may need to patch deliberately. The +# same SARIF file is summarized into a review_status for the unified PR comment. on: workflow_call: @@ -42,63 +41,80 @@ on: workflow_dispatch: permissions: - # Required to upload SARIF file to CodeQL. See: https://github.com/github/codeql-action/issues/2117 - actions: read contents: read - security-events: write jobs: - scan: - name: Scan lockfiles - uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@baa4139e56d6312335d899e6ba045fa16d1d3d0b # v2.5.1 - with: - # Scan explicit lockfiles rather than recursing, so we only look at - # the five sources of truth and skip vendored / test / worktree dirs. - scan-args: |- - --lockfile=uv.lock - --lockfile=package-lock.json - --lockfile=website/package-lock.json - --lockfile=plugins/platforms/photon/sidecar/package-lock.json - --lockfile=scripts/whatsapp-bridge/package-lock.json - # The upstream reusable workflow uploads this exact file under its - # fixed artifact name, which the wrapper downloads below. - results-file-name: osv-results.sarif - # Direct schedule/push/dispatch runs do not populate workflow_call - # inputs. In a called workflow github.event_name remains the caller's - # original event, so exclude both PR and workflow_call contexts here. - upload-sarif: ${{ inputs.upload-sarif || (github.event_name != 'pull_request' && github.event_name != 'workflow_call') }} - fail-on-vuln: false - emit-status: - name: Emit review status + name: Scan lockfiles and emit review status + permissions: + actions: read + contents: read + security-events: write runs-on: ubuntu-latest - needs: scan - if: always() outputs: review_status: ${{ steps.emit.outputs.review_status }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Download SARIF result - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + - name: Prepare OSV output directory + run: mkdir -p osv-results + + - name: Run OSV scanner + uses: google/osv-scanner-action/osv-scanner-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 with: - name: OSV Scanner SARIF file - path: /tmp/osv-results + # Scan explicit lockfiles rather than recursing, so we only look at + # the five sources of truth and skip vendored / test / worktree dirs. + scan-args: |- + --output=osv-results/results.json + --format=json + --lockfile=uv.lock + --lockfile=package-lock.json + --lockfile=website/package-lock.json + --lockfile=plugins/platforms/photon/sidecar/package-lock.json + --lockfile=scripts/whatsapp-bridge/package-lock.json continue-on-error: true + - name: Build OSV SARIF + uses: google/osv-scanner-action/osv-reporter-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 + with: + scan-args: |- + --output=osv-results/osv-results.sarif + --new=osv-results/results.json + --gh-annotations=false + --fail-on-vuln=false + + - name: Upload SARIF result artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: OSV Scanner SARIF file + path: osv-results/osv-results.sarif + retention-days: 5 + if-no-files-found: error + + - name: Upload to code scanning + # inputs.upload-sarif is unset (not false) on the schedule/push/workflow_dispatch + # triggers below, where the upload must still happen — only an explicit `false` + # (ci.yaml's PR call) skips it. + if: ${{ !cancelled() && inputs.upload-sarif != false }} + uses: github/codeql-action/upload-sarif@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v4.31.10 + with: + sarif_file: osv-results/osv-results.sarif + - name: Emit review_status id: emit + if: always() run: | set -euo pipefail STATUS="[]" - if [ -f /tmp/osv-results/osv-results.sarif ]; then + if [ -f osv-results/osv-results.sarif ]; then # Count vulnerabilities from the SARIF file VULN_COUNT=$(python3 -c " import json, sys try: - with open('/tmp/osv-results/osv-results.sarif') as f: + with open('osv-results/osv-results.sarif') as f: data = json.load(f) count = 0 vulns = [] @@ -122,7 +138,7 @@ jobs: VULN_DETAIL=$(python3 -c " import json, sys try: - with open('/tmp/osv-results/osv-results.sarif') as f: + with open('osv-results/osv-results.sarif', encoding='utf-8') as f: data = json.load(f) vulns = [] for run in data.get('runs', []): diff --git a/.github/workflows/skills-index-freshness.yml b/.github/workflows/skills-index-freshness.yml index ff2bc393aa28a..6da16b5a90b1e 100644 --- a/.github/workflows/skills-index-freshness.yml +++ b/.github/workflows/skills-index-freshness.yml @@ -124,6 +124,7 @@ jobs: with: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-issues: write - name: Open issue on degraded / failed probe if: steps.probe.outputs.status != 'ok' diff --git a/.github/workflows/skills-index.yml b/.github/workflows/skills-index.yml index cf6812630ebdd..eae459986f19b 100644 --- a/.github/workflows/skills-index.yml +++ b/.github/workflows/skills-index.yml @@ -13,7 +13,6 @@ on: permissions: contents: read - actions: write # to trigger deploy-site.yml on schedule jobs: build-index: @@ -31,6 +30,7 @@ jobs: with: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: read - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -59,6 +59,9 @@ jobs: trigger-deploy: needs: build-index if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + permissions: + actions: write # trigger deploy-site.yml on schedule or manual refresh + contents: read # check out the local token-minting action runs-on: ubuntu-latest timeout-minutes: 15 environment: trusted-automation @@ -75,6 +78,7 @@ jobs: with: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-actions: write - name: Trigger Deploy Site workflow env: diff --git a/.github/workflows/supply-chain-audit.yml b/.github/workflows/supply-chain-audit.yml index d73b749ab489e..ba8ef936d1e09 100644 --- a/.github/workflows/supply-chain-audit.yml +++ b/.github/workflows/supply-chain-audit.yml @@ -38,6 +38,10 @@ on: description: Whether pyproject.toml changed. type: boolean required: true + npm_lock: + description: Whether package-lock.json changed. + type: boolean + required: true outputs: review_status: description: JSON array of review status objects for the review comment assembler. @@ -47,8 +51,8 @@ on: value: ${{ jobs.aggregate.outputs.critical_findings }} permissions: - pull-requests: write contents: read + pull-requests: read jobs: scan: @@ -255,9 +259,94 @@ jobs: echo "::error::PyPI dependencies without upper bounds detected. Add /dev/null || true) + dirs=$(printf '%s\n' $changed | sed 's|/package-lock.json||; s|^$|.|' | sort -u | tr '\n' ' ') + echo "dirs=${dirs:-.}" >> "$GITHUB_OUTPUT" + + - name: Install dependencies without lifecycle scripts + run: | + for dir in ${{ steps.lockdirs.outputs.dirs }}; do + (cd "$dir" && npm ci --ignore-scripts) || true + done + + - name: Run npm high audit + id: audit + run: | + set +e + rc=0 + for dir in ${{ steps.lockdirs.outputs.dirs }}; do + (cd "$dir" && npm audit --audit-level=high --json) >> npm-audit.json || rc=$? + done + echo "exit_code=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Emit review_status + id: emit-status + if: always() + env: + AUDIT_EXIT_CODE: ${{ steps.audit.outputs.exit_code }} + run: | + python3 - <<'PYEOF' + import json, os + + failed = os.environ.get("AUDIT_EXIT_CODE") not in ("0", "") + status = [] + if failed: + with open("npm-audit.json", encoding="utf-8") as f: + audit = json.load(f) + vulns = audit.get("metadata", {}).get("vulnerabilities", {}) + high = vulns.get("high", 0) + critical = vulns.get("critical", 0) + names = sorted(audit.get("vulnerabilities", {}).keys()) + status = [{ + "source": "supply chain", + "results": [{ + "kind": "action_required", + "title": "High-severity npm advisories", + "summary": f"npm audit found {high} high and {critical} critical advisories.", + "detail": "Affected packages: " + ", ".join(names[:25]), + "how_to_fix": "Update the affected direct pins or root overrides, regenerate package-lock.json, and rerun npm audit --audit-level=high." + }] + }] + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"review_status={json.dumps(status)}\n") + PYEOF + + - name: Fail on high-severity npm advisories + if: steps.audit.outputs.exit_code != '0' + run: | + cat npm-audit.json + echo "::error::npm audit found high-severity advisories. Update pins/overrides and package-lock.json." + exit 1 + aggregate: name: Aggregate review statuses - needs: [scan, dep-bounds] + needs: [scan, dep-bounds, npm-audit] if: always() runs-on: ubuntu-latest timeout-minutes: 15 @@ -270,13 +359,14 @@ jobs: env: SCAN_STATUS: ${{ needs.scan.outputs.review_status }} DEP_STATUS: ${{ needs.dep-bounds.outputs.review_status }} + NPM_STATUS: ${{ needs.npm-audit.outputs.review_status }} CRITICAL_FINDINGS: ${{ needs.scan.outputs.critical_findings }} run: | python3 - <<'PYEOF' import json, os merged = [] - for key in ("SCAN_STATUS", "DEP_STATUS"): + for key in ("SCAN_STATUS", "DEP_STATUS", "NPM_STATUS"): raw = os.environ.get(key, "") if not raw: continue diff --git a/.github/workflows/windows-venv-e2e.yml b/.github/workflows/windows-venv-e2e.yml index a09b2f2d45377..878f956d8db56 100644 --- a/.github/workflows/windows-venv-e2e.yml +++ b/.github/workflows/windows-venv-e2e.yml @@ -15,7 +15,8 @@ name: Windows venv-holder live E2E on: push: branches: - - "wine2e/**" + - 'wine2e/**' + workflow_dispatch: permissions: contents: read diff --git a/.gitignore b/.gitignore index 7cf39fcfc9df8..7de6d08c7b852 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store /venv/ +/venv /venv.old/ /venv.stale.runtime-*/ /bin/ @@ -143,6 +144,7 @@ models-dev-upstream/ .codex/ .cursor/ .gemini/ +.serena/ .zed/ .mcp.json opencode.json diff --git a/.importlinter b/.importlinter new file mode 100644 index 0000000000000..0a9e9b09833b1 --- /dev/null +++ b/.importlinter @@ -0,0 +1,51 @@ +[importlinter] +root_packages = + agent + gateway + hermes_cli + plugins + tools + +[importlinter:contract:tool-registry-narrow-waist] +name = Tool registry remains below agent and delivery layers +type = forbidden +source_modules = + tools.registry +forbidden_modules = + hermes_cli + plugins +allow_indirect_imports = true + +[importlinter:contract:profile-secret-scope-leaf] +name = Profile secret scope stays independent of runtime layers +type = forbidden +source_modules = + agent.secret_scope +forbidden_modules = + gateway + plugins + tools +allow_indirect_imports = true + +[importlinter:contract:gateway-session-context-leaf] +name = Gateway session context stays transitively dependency-light +type = forbidden +source_modules = + gateway.session_context +forbidden_modules = + hermes_cli + plugins + tools +allow_indirect_imports = false + +[importlinter:contract:plugin-capability-consent-leaf] +name = Plugin capability consent stays independent of runtime layers +type = forbidden +source_modules = + hermes_cli.plugin_capabilities +forbidden_modules = + agent + gateway + plugins + tools +allow_indirect_imports = true diff --git a/.mailmap b/.mailmap index 5af0a658645a4..92b91cef4b47e 100644 --- a/.mailmap +++ b/.mailmap @@ -17,6 +17,8 @@ Teknium <127238744+teknium1@users.noreply.github.com> # === Contributors — personal/work emails mapped to GitHub noreply === # Format: Canonical Name +Mike DeMott <25466867+mrkillbob@users.noreply.github.com> + # Verified via GH API email search kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> luyao618 <364939526@qq.com> <364939526@qq.com> diff --git a/.npmrc b/.npmrc index 0e252084614fe..c6ca2711c5fe3 100644 --- a/.npmrc +++ b/.npmrc @@ -34,7 +34,8 @@ min-release-age-exclude[]=minimatch # brace-expansion 5.0.9 includes fixes for vulns. remove this when 5.0.9 is > 2wks old min-release-age-exclude[]=brace-expansion -# js-yaml 4.3.1 includes fixes for GHSA-5p4m-2wfm-xmqj. remove when > 2wks old (rel 2026-07-31) +# js-yaml 4.3.2 includes fixes for GHSA-5p4m-2wfm-xmqj and GHSA-2883-xcg3-v3hh. +# remove when > 2wks old (rel 2026-08-26) min-release-age-exclude[]=js-yaml # nanoid 3.3.17 includes fixes for GHSA-2v37-7h3g-55p8. remove when > 2wks old (rel 2026-08-03) @@ -61,3 +62,17 @@ min-release-age-exclude[]=@oxc-project/types # ink needs min-release-age-exclude[]=lightningcss min-release-age-exclude[]=postcss + +# electron-builder 26.16.1 (rel 2026-09-07) fixes high-severity transitive advisories in its +# app-builder-lib/builder-util/dmg-builder/electron-publish/js-yaml dependency chain +# (npm audit high). remove this when 26.16.1 is > 2wks old. +min-release-age-exclude[]=electron-builder +min-release-age-exclude[]=app-builder-lib +min-release-age-exclude[]=builder-util +min-release-age-exclude[]=dmg-builder +min-release-age-exclude[]=electron-publish +min-release-age-exclude[]=electron-builder-squirrel-windows + +# @xmldom/xmldom fix for multiple high-severity injection/ReDoS advisories (npm audit high). +# remove this when the fix is > 2wks old. +min-release-age-exclude[]=@xmldom/xmldom diff --git a/.python-version b/.python-version index 2c0733315e415..64af02e6251aa 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11 +3.13.6 diff --git a/AGENTS.md b/AGENTS.md index aecd44860c669..3423b08e24d7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,24 @@ # Hermes Agent - Development Guide +## Local project environment + +This checkout is the separate Hermes-agent project and targets Python 3.13.6. Use +`./.venv/bin/python` for the primary Hermes-agent checkout. The canonical environment +is managed outside the checkout and is shared only by Hermes-agent worktrees. New +Hermes-agent Git worktrees created by the task, conversation, +subagent, web-Git, CLI, or PR-maintenance paths receive a `.venv` link to the verified +repository runtime before an agent is released; inside those worktrees use +`./.venv/bin/python` (which must resolve to Python 3.13.6). Do not install Hermes dependencies into the LunaBot environment at +`/Users/mikedemott/LunaBot-default/.venv`; that environment belongs to LunaBot/TradingBotV18. + + +## VS Code Studio Access + +- Shared launcher: `/Users/mikedemott/.local/bin/vscode-studio`. +- Open this canonical Hermes workspace with `vscode-studio hermes`. +- Open only curated workspaces or specific files; do not open `/Users/mikedemott/Codex` or network drives from Hermes agents. +- VS Code is for inspection/editing ergonomics only; tests and runtime evidence still come from explicit commands. + Instructions for AI coding assistants and developers working on the hermes-agent codebase. This root file holds only what applies everywhere. Each area has its own `AGENTS.md` (aim for ~8k chars; `agent/subdirectory_hints.py` delivers up to 32k and truncates head/tail with a warning diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e6761ee18a46..4c1ae514256b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,7 +109,7 @@ A well-built third-party-product plugin can clear automated review and still be | Requirement | Notes | |-------------|-------| | **Git** | With the `git-lfs` extension installed | -| **Python 3.11–3.13** | uv will install it if missing | +| **Python 3.13.6** | uv will install it if missing | | **uv** | Fast Python package manager ([install](https://docs.astral.sh/uv/)) | | **Node.js 20+** | Optional — needed for browser tools and WhatsApp bridge (matches root `package.json` engines) | @@ -159,8 +159,8 @@ tree means no relative path from the workspace resolves to it. git clone https://github.com/NousResearch/hermes-agent.git cd hermes-agent -# Create venv with Python 3.11, OUTSIDE the source tree -uv venv ~/.hermes/venvs/hermes-dev --python 3.11 +# Create venv with Python 3.13.6, OUTSIDE the source tree +uv venv ~/.hermes/venvs/hermes-dev --python 3.13.6 export VIRTUAL_ENV="$HOME/.hermes/venvs/hermes-dev" export PATH="$VIRTUAL_ENV/bin:$PATH" diff --git a/README.md b/README.md index c05112266746f..2c3d1d455b079 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Run this in PowerShell: iex (irm https://hermes-agent.nousresearch.com/install.ps1) ``` -The installer handles everything: uv, Python 3.11, Node.js, ripgrep, ffmpeg, **and a portable Git Bash** (MinGit, unpacked to `%LOCALAPPDATA%\hermes\git` — no admin required, completely isolated from any system Git install). Hermes uses this bundled Git Bash to run shell commands. +The installer handles everything: uv, Python 3.13.6, Node.js, ripgrep, ffmpeg, **and a portable Git Bash** (MinGit, unpacked to `%LOCALAPPDATA%\hermes\git` — no admin required, completely isolated from any system Git install). Hermes uses this bundled Git Bash to run shell commands. If you already have Git installed, the installer detects it and uses that instead. Otherwise a ~45MB MinGit download is all you need — it won't touch or interfere with any system Git. @@ -239,7 +239,7 @@ against its own checkout, destroying the running runtime mid-session. ```bash curl -LsSf https://astral.sh/uv/install.sh | sh -uv venv ~/.hermes/venvs/hermes-dev --python 3.11 +uv venv ~/.hermes/venvs/hermes-dev --python 3.13.6 source ~/.hermes/venvs/hermes-dev/bin/activate uv pip install -e ".[all,dev]" scripts/run_tests.sh diff --git a/agent/activity_tracking.py b/agent/activity_tracking.py index 0f2a38520887c..47d559519a89a 100644 --- a/agent/activity_tracking.py +++ b/agent/activity_tracking.py @@ -75,7 +75,11 @@ def _touch_activity( from tools.kanban_tools import ( heartbeat_current_worker_from_env, inject_new_comments_from_env ) - heartbeat_current_worker_from_env() + from agent.interrupt_compat import request_hard_interrupt + heartbeat_current_worker_from_env(on_lease_lost=lambda tid: request_hard_interrupt( + self, f"Kanban lease lost for {tid}; stopping superseded worker.", + tool_reason="kanban lease lost", + )) # Fold new operator notes into the running turn (OUT-OF-BAND steer). inject_new_comments_from_env(self) if force_persist: diff --git a/agent/agent_init.py b/agent/agent_init.py index 30157f735ff28..ade3285b306b1 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1016,6 +1016,8 @@ def _client_kwargs_from_routed(client, timeout) -> Dict[str, Any]: def _fallback_entries(fallback_model) -> List[Dict[str, Any]]: """Normalize legacy single-dict ``fallback_model`` / list ``fallback_providers``.""" + if os.environ.get("HERMES_KANBAN_LOCAL_ONLY") == "1": + return [] if isinstance(fallback_model, dict): fallback_model = [fallback_model] if not isinstance(fallback_model, list): diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 9f77e631a1d73..f3805797222d4 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -107,6 +107,7 @@ def aux_probe_mode(): from agent.credential_pool import load_pool +from agent.llm_egress_firewall import EgressBlocked from agent.model_metadata import ( MINIMUM_CONTEXT_LENGTH, get_model_context_length, strip_codex_context_variant_suffix as _strip_codex_ctx_variant, @@ -2357,6 +2358,120 @@ def _read_main_api_key_if_same_host(aux_base_url: str) -> str: contextvars.ContextVar("auxiliary_relay_call", default=None) ) +def _auxiliary_egress_binding( + client: Any, + *, + provider: str | None, + model: str | None, + api_mode: str | None, +) -> tuple[Any, Any] | None: + """Build the complete identity and route for protected auxiliary calls. + + Protected exactly like the main request path (`authorize_agent_sdk_kwargs`): + an exact firewall-owning provider (anthropic/openai-codex/nous/nous-portal/ + nousresearch), OR every provider when ``HERMES_KANBAN_PROTECTED_REMOTE=1`` -- + a compression/review/vision auxiliary call inside a protected Kanban task is + just as much an egress point as the main request, and previously skipped + authorization/sanitization entirely whenever it used a non-firewall provider. + """ + from agent.llm_egress_runtime import provider_uses_egress_firewall + + normalized_provider = _normalize_aux_provider(provider) + protected_remote_marker = os.environ.get("HERMES_KANBAN_PROTECTED_REMOTE") == "1" + if not protected_remote_marker and not provider_uses_egress_firewall(normalized_provider): + return None + from agent.source_provenance import DEFAULT_POLICY_DIGEST + + runtime = _normalize_main_runtime(None) + raw_runtime = _RUNTIME_MAIN_CONTEXT.get() or {} + relay = _RELAY_AUX_CALL_CONTEXT.get() or {} + request_id = str(relay.get("request_id") or f"aux-{uuid.uuid4().hex}") + session_id = str( + runtime.get("session_id") + or raw_runtime.get("session_id") + or f"aux-session:{request_id}" + ) + turn_id = str( + raw_runtime.get("turn_id") + or f"{session_id}:aux:{str(relay.get('task') or 'call')}" + ) + policy_digest = str( + raw_runtime.get("policy_digest") + or raw_runtime.get("llm_egress_policy_digest") + or DEFAULT_POLICY_DIGEST + ) + candidate_base_url = getattr(client, "base_url", "") + if not isinstance(candidate_base_url, str) or not candidate_base_url.startswith( + ("http://", "https://") + ): + candidate_base_url = raw_runtime.get("base_url") + if not isinstance(candidate_base_url, str) or not candidate_base_url.startswith( + ("http://", "https://") + ): + if normalized_provider == "openai-codex": + candidate_base_url = "https://chatgpt.com/backend-api/codex" + elif normalized_provider == "anthropic": + candidate_base_url = "https://api.anthropic.com/v1" + else: + candidate_base_url = _NOUS_DEFAULT_BASE_URL + base_url = candidate_base_url + resolved_api_mode = str( + api_mode + or ( + "codex_responses" + if normalized_provider == "openai-codex" + else "chat_completions" + ) + ) + agent_attrs = { + "provider": normalized_provider, + "model": str(model or ""), + "base_url": base_url, + "api_mode": resolved_api_mode, + "session_id": session_id, + "_current_turn_id": turn_id, + "_current_api_request_id": request_id, + "_llm_egress_policy_digest": policy_digest, + "_llm_egress_state_dir": Path(get_hermes_home()) / "egress", + } + if str(relay.get("task") or "") == "compression": + agent_attrs.update( + _llm_egress_max_serialized_bytes=2_000_000, + _llm_egress_max_conservative_tokens=666_667, + _llm_egress_max_sanitized_bytes=2_000_000, + _llm_egress_max_sanitized_segment_bytes=32_768, + _llm_egress_max_granted_serialized_bytes=2_000_000, + _llm_egress_max_granted_conservative_tokens=666_667, + ) + agent = SimpleNamespace(**agent_attrs) + route = SimpleNamespace( + provider=normalized_provider, + model=str(model or ""), + base_url=base_url, + api_mode=resolved_api_mode, + ) + return agent, route + + +def _dispatch_auxiliary_request( + client: Any, + request: dict[str, Any], + callback: Callable[[dict[str, Any]], Any], + *, + provider: str | None, + model: str | None, + api_mode: str | None, +) -> Any: + binding = _auxiliary_egress_binding( + client, provider=provider, model=model or request.get("model"), api_mode=api_mode + ) + if binding is None: + return callback(request) + from agent.llm_egress_runtime import dispatch_authorized_agent_request + + agent, route = binding + return dispatch_authorized_agent_request(agent, request, callback, route=route) + @contextlib.contextmanager def _relay_aux_call_scope(args: tuple, kwargs: dict): @@ -2444,6 +2559,15 @@ def _relay_sync_completion( kwargs = prepare_chat_messages(client, kwargs) callback = create or (lambda request: client.chat.completions.create(**request)) + raw_callback = callback + callback = lambda request: _dispatch_auxiliary_request( + client, + request, + raw_callback, + provider=provider, + model=request.get("model"), + api_mode=api_mode, + ) route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) # Isolate only the provider callback so the owning thread can unwind its lease/DB # transaction on hard cancel without touching the shared client. @@ -2466,6 +2590,29 @@ async def _relay_async_completion( kwargs = prepare_chat_messages(client, kwargs) callback = create or (lambda request: client.chat.completions.create(**request)) + raw_callback = callback + + async def _authorized_callback(request: dict[str, Any]) -> Any: + binding = _auxiliary_egress_binding( + client, + provider=provider, + model=request.get("model"), + api_mode=api_mode, + ) + if binding is None: + return await raw_callback(request) + from agent.llm_egress_runtime import dispatch_authorized_agent_request + + agent, route = binding + result = dispatch_authorized_agent_request( + agent, + request, + raw_callback, + route=route, + ) + return await result if inspect.isawaitable(result) else result + + callback = _authorized_callback route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) if route is None: return await callback(kwargs) @@ -2483,9 +2630,17 @@ def _relay_sync_stream( from agent.auxiliary_wire import prepare_chat_messages kwargs = prepare_chat_messages(client, kwargs) + callback = lambda request: _dispatch_auxiliary_request( + client, + request, + lambda authorized: client.chat.completions.create(**authorized), + provider=provider, + model=kwargs.get("model"), + api_mode=api_mode, + ) route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) if route is None: - return client.chat.completions.create(**kwargs) + return callback(kwargs) provider_name, fallback_model, metadata = route from agent import relay_llm return relay_llm.stream_current( @@ -2849,7 +3004,13 @@ def _try_anthropic(explicit_api_key: str = None) -> Tuple[Optional[Any], Optiona _MAIN_RUNTIME_FIELDS = ("provider", "model", "base_url", "api_key", "api_mode", "auth_mode") -_MAIN_RUNTIME_CONTEXT_FIELDS = _MAIN_RUNTIME_FIELDS + ("requested_provider",) +_MAIN_RUNTIME_CONTEXT_FIELDS = _MAIN_RUNTIME_FIELDS + ( + "requested_provider", + "session_id", + "turn_id", + "policy_digest", + "llm_egress_policy_digest", +) def _normalize_main_runtime(main_runtime: Optional[Dict[str, Any]]) -> Dict[str, Any]: @@ -3068,6 +3229,20 @@ def _is_transient_transport_error(exc: Exception) -> bool: return isinstance(status, int) and (status == 408 or 500 <= status < 600) +def _is_safe_egress_fallback_candidate(client: Any, provider: str, model: str) -> bool: + """Permit a blocked remote request to fall back only to local execution. + + A blocked payload must never be retried against another remote provider; + only an explicitly local/loopback fallback is eligible, and its callback + still receives a fresh request copy at the local boundary. + """ + from agent.llm_egress_firewall import DestinationClass, classify_destination + + base_url = str(getattr(client, "base_url", "") or "") + destination = classify_destination(provider, base_url, "chat_completions") + return destination in {DestinationClass.LOCAL_PROCESS, DestinationClass.LOOPBACK} + + _DEFAULT_TRANSIENT_RETRIES = 2 _TRANSIENT_RETRY_BACKOFF_BASE = 1.0 # Backoff base (seconds); overridable so tests can zero it out. @@ -7014,6 +7189,7 @@ def call_llm( prior_progress_hook = getattr(_aux_progress, "hook", None) try: with ( + scoped_runtime_main(main_runtime), aux_progress_hook( prior_progress_hook if callable(prior_progress_hook) @@ -7193,6 +7369,29 @@ def _primary(**validate_kw: Any) -> Any: except Exception as transient_err: if not _should_retry_same_provider(task, transient_err, ""): raise + # Compression is on the critical preflight path: a user cannot + # continue or resume an oversized session until it compacts. A + # same-provider retry on a timeout means another full ``timeout``- + # long wall-clock block before the except-chain below can fall + # back — doubling the user-visible stall (issue #54465). Skip the + # same-provider retry for compression on a full-budget timeout and + # fall straight through to provider/model fallback; fast blips (a + # streaming-close or a 5xx) still retry, since those are cheap. + if task == "compression" and _is_timeout_error(transient_err): + # A fast first-token fail (dead stream detected within the + # 60s no-progress window, zero output seen) is cheap — take + # the normal same-provider retry chain first; the provider + # is often fine and only that one stream was stillborn. A + # mid-stream stall or hard-ceiling timeout skips straight to + # fallback, because re-running a multi-minute summary on the + # same provider doubles the user-visible stall (#54465). + if "no-progress timeout" not in str(transient_err): + logger.info( + "Auxiliary compression: timeout on the critical path; " + "skipping same-provider retry and falling back: %s", + transient_err, + ) + raise _max_transient_retries = _transient_retry_count() _last_transient = transient_err for _attempt in range(1, _max_transient_retries + 1): diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 184b71d69edf7..4db6332b42059 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -675,6 +675,46 @@ def _bedrock_converse_call(api_kwargs: dict, *, stream: bool, on_stream_denied=N return finish(raw_response) +_EGRESS_PROTECTED_PROVIDERS = frozenset( + {"anthropic", "openai-codex", "nous", "nous-portal", "nousresearch"} +) + + +def _destination_requires_egress_firewall(agent) -> bool: + """Return whether this route is under the protected egress contract.""" + + provider = str(getattr(agent, "provider", "") or "").strip().lower() + return provider in _EGRESS_PROTECTED_PROVIDERS or ( + os.environ.get("HERMES_KANBAN_PROTECTED_REMOTE") == "1" + ) + + +def _attach_source_provenance_sidecar( + agent, kwargs: dict, messages: list | None = None, *, sidecar: list | None = None +) -> dict: + """Carry internal read proofs around strict wire-message conversion.""" + + if not _destination_requires_egress_firewall(agent): + return kwargs + from agent.source_provenance_tools import build_source_provenance_sidecar + + if sidecar is None: + sidecar = build_source_provenance_sidecar(messages) + if not sidecar: + return kwargs + return {**kwargs, "_hermes_source_provenance": sidecar} + + +def _dispatch_provider_request(agent, request, callback): + """Apply the exact provider-bound egress policy at a physical call site.""" + + if not _destination_requires_egress_firewall(agent): + return callback(request) + from agent.llm_egress_runtime import dispatch_authorized_agent_request + + return dispatch_authorized_agent_request(agent, request, callback) + + def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): """Run one non-streaming LLM request for the active api_mode and return it. @@ -684,13 +724,22 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): manage their own clients. Interrupt/abort/close semantics stay in callers. """ if agent.api_mode == "codex_responses": - return agent._run_codex_stream(api_kwargs, client=make_client("codex_stream_request"), - on_first_delta=getattr(agent, "_codex_on_first_delta", None)) + codex_client = make_client("codex_stream_request") + return _dispatch_provider_request( + agent, api_kwargs, + lambda authorized: agent._run_codex_stream( + authorized, client=codex_client, + on_first_delta=getattr(agent, "_codex_on_first_delta", None), + ), + ) if agent.api_mode == "anthropic_messages": # Request-local client so the stale/interrupt watchdog aborts sockets # from the stranger thread while the worker owns the SDK close (#67142). request_client = make_client("anthropic_messages_request", kind="anthropic_messages") - return agent._anthropic_messages_create(api_kwargs, client=request_client) + return _dispatch_provider_request( + agent, api_kwargs, + lambda authorized: agent._anthropic_messages_create(authorized, client=request_client), + ) if agent.api_mode == "bedrock_converse": return _bedrock_converse_call(api_kwargs, stream=False) if agent.provider == "moa": @@ -702,8 +751,15 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): _completions = getattr(getattr(agent.client, "chat", None), "completions", None) if not callable(getattr(_completions, "prepare", None)): api_kwargs.pop("_moa_prepared_request", None) - return agent.client.chat.completions.create(**api_kwargs) - return make_client("chat_completion_request").chat.completions.create(**api_kwargs) + return _dispatch_provider_request( + agent, api_kwargs, + lambda authorized: agent.client.chat.completions.create(**authorized), + ) + request_client = make_client("chat_completion_request") + return _dispatch_provider_request( + agent, api_kwargs, + lambda authorized: request_client.chat.completions.create(**authorized), + ) def should_use_direct_api_call(agent) -> bool: @@ -1379,6 +1435,12 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non def _build_api_kwargs_for_mode(agent, api_messages: list, tools_for_api: list | None = None) -> dict: + # Capture internal provenance before any transport converts or sanitizes + # messages. Codex Responses removes internal tool-message keys entirely, + # and some chat transports normalize the list in place. + from agent.source_provenance_tools import build_source_provenance_sidecar + + _source_sidecar = build_source_provenance_sidecar(api_messages) # One-shot continuation override — consumed exactly once, on the FIRST # request this call builds (only one api_mode branch runs per invocation). reasoning_config = _reasoning_config_for_wire(agent) @@ -1388,14 +1450,28 @@ def _build_api_kwargs_for_mode(agent, api_messages: list, tools_for_api: list | # in agent.request_overrides; auto/cold windows layer the fast override per request. request_overrides = effective_request_overrides(agent) if agent.api_mode == "anthropic_messages": - return _build_anthropic_kwargs(agent, api_messages, tools_for_api, reasoning_config, request_overrides) - if agent.api_mode == "bedrock_converse": - return _build_bedrock_kwargs(agent, api_messages, tools_for_api) - # Rotation-stable logical cache scope shared by every OpenAI-wire branch - # (memoized on the agent); anthropic/bedrock above don't use it. - cache_scope_id = _prompt_cache_scope_for_agent(agent) - builder = _build_codex_kwargs if agent.api_mode == "codex_responses" else _build_chat_completions_kwargs - return builder(agent, api_messages, tools_for_api, reasoning_config, request_overrides, cache_scope_id) + api_kwargs = _build_anthropic_kwargs(agent, api_messages, tools_for_api, reasoning_config, request_overrides) + elif agent.api_mode == "bedrock_converse": + api_kwargs = _build_bedrock_kwargs(agent, api_messages, tools_for_api) + else: + # Rotation-stable logical cache scope shared by every OpenAI-wire branch + # (memoized on the agent); anthropic/bedrock above don't use it. + cache_scope_id = _prompt_cache_scope_for_agent(agent) + builder = _build_codex_kwargs if agent.api_mode == "codex_responses" else _build_chat_completions_kwargs + api_kwargs = builder(agent, api_messages, tools_for_api, reasoning_config, request_overrides, cache_scope_id) + # Provider-owned capability boundary, applied last so it can strip anything + # request_overrides just added that the verified route/model can't accept. + from providers import get_provider_profile + + provider_profile = get_provider_profile(agent.provider) + if provider_profile is not None: + _supports_reasoning_fn = getattr(agent, "_supports_reasoning_extra_body", None) + api_kwargs = provider_profile.sanitize_request_kwargs( + api_kwargs, agent=agent, + supports_reasoning=_supports_reasoning_fn() if callable(_supports_reasoning_fn) else False, + base_url=getattr(agent, "base_url", None), + ) + return _attach_source_provenance_sidecar(agent, api_kwargs, sidecar=_source_sidecar) def _model_dump_safe(obj): @@ -1637,10 +1713,47 @@ def _fallback_entry_unavailable_without_network(agent, fb: dict) -> Optional[str FailoverReason.long_context_tier: "long-context tier unavailable", FailoverReason.oauth_long_context_beta_forbidden: "OAuth long-context beta unavailable", FailoverReason.llama_cpp_grammar_pattern: "grammar pattern rejected", + FailoverReason.egress_policy_blocked: "local egress policy blocked the request", FailoverReason.unknown: "provider failure", } +def _fallback_destination_class(fb: dict): + """Resolve a fallback's configured destination for egress-aware routing. + + Egress policy failures must never walk another remote provider with the + same unsafe request. A fallback entry may omit ``base_url`` and rely on + the provider definition in config.yaml, so resolve that URL here rather + than trusting the provider label (provider names are not a security + boundary). + """ + from agent.llm_egress_firewall import classify_destination + + base_url = (fb.get("base_url") or "").strip() + if not base_url: + try: + from hermes_cli.config import load_config + + provider_cfg = (load_config() or {}).get("providers", {}).get( + (fb.get("provider") or "").strip(), {} + ) + if isinstance(provider_cfg, dict): + base_url = str( + provider_cfg.get("api") + or provider_cfg.get("base_url") + or "" + ).strip() + except Exception: + # Unknown destination must remain unknown and therefore cannot + # inherit local trust after an egress policy rejection. + base_url = "" + return classify_destination( + str(fb.get("provider") or ""), + base_url, + fb.get("api_mode") or "chat_completions", + ) + + def _fallback_reason_text(reason: "FailoverReason | None") -> str: """Return a concise operator-facing explanation for a fallback switch.""" label = _FALLBACK_REASON_LABELS.get(reason) @@ -1723,14 +1836,27 @@ def _fallback_chain_exhausted(agent, reason: "FailoverReason | None") -> bool: return False -def _should_skip_fallback_candidate(agent, fb: dict, fb_key: tuple, fb_provider: str, fb_model: str, unavailable: set) -> bool: - """True when the entry is already unavailable, malformed, locally unusable, or resolves - to the backend that just failed (falling back to it would loop the failure).""" +def _should_skip_fallback_candidate( + agent, fb: dict, fb_key: tuple, fb_provider: str, fb_model: str, unavailable: set, + reason: "FailoverReason | None" = None, +) -> bool: + """True when the entry is already unavailable, malformed, locally unusable, resolves + to the backend that just failed (falling back to it would loop the failure), or — for + an egress-policy rejection — is itself a remote destination (retrying the same unsafe + payload against another remote provider cannot succeed and only adds noise).""" if fb_key in unavailable: logger.debug("Fallback skip: %s previously marked unavailable", fb_key) return True if not fb_provider or not fb_model: return True + if reason == FailoverReason.egress_policy_blocked: + from agent.llm_egress_firewall import DestinationClass + destination = _fallback_destination_class(fb) + if destination not in (DestinationClass.LOCAL_PROCESS, DestinationClass.LOOPBACK): + logger.warning( + "Fallback skip: %s/%s is a remote destination (%s); egress policy blocked the " + "current request, so only local fallbacks are eligible", fb_provider, fb_model, destination) + return True local_skip_reason = _fallback_entry_unavailable_without_network(agent, fb) if local_skip_reason: unavailable.add(fb_key) @@ -1825,6 +1951,10 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool """Switch to the next fallback model/provider in the chain; False when exhausted. Swaps client, model slug and provider in place so the retry loop continues on the new backend; client construction goes through resolve_provider_client (no duplicated provider→key mappings).""" + if reason == FailoverReason.unsupported_thinking: + # The model itself lacks thinking support — every remote candidate in the chain is + # configured the same way a human picked, so walking it won't fix a capability gap. + return False from agent.fallback_cooldown import _arm_rate_limit_cooldown cooldown_seconds = _arm_rate_limit_cooldown(agent, reason) while True: @@ -1838,7 +1968,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool unavailable = agent._unavailable_fallback_keys fb_provider = (fb.get("provider") or "").strip().lower() fb_model = (fb.get("model") or "").strip() - if _should_skip_fallback_candidate(agent, fb, fb_key, fb_provider, fb_model, unavailable): + if _should_skip_fallback_candidate(agent, fb, fb_key, fb_provider, fb_model, unavailable, reason=reason): continue try: @@ -2015,8 +2145,29 @@ def _iteration_summary_chat_kwargs(agent, api_messages: list) -> dict: is_lmstudio = provider_name == "lmstudio" and agent._supports_reasoning_extra_body() lm_reasoning_effort = agent._resolve_lmstudio_summary_reasoning_effort() if is_lmstudio else None + # Resolved once and reused below for both the reasoning-extras hook (capability-gated + # omission, e.g. a Nous model that 400s on a disabled reasoning config) and build_extra_body. + provider_profile = None + with contextlib.suppress(Exception): + from providers import get_provider_profile + provider_profile = get_provider_profile(agent.provider) + extra_body = {} - if not is_lmstudio and agent._supports_reasoning_extra_body(): + top_level_from_profile = {} + handles_reasoning = False + if provider_profile is not None: + _supports_reasoning = agent._supports_reasoning_extra_body() + reasoning_extra, top_level_from_profile = provider_profile.build_api_kwargs_extras( + reasoning_config=agent.reasoning_config, supports_reasoning=_supports_reasoning, + model=agent.model, base_url=agent.base_url, + ) + extra_body.update(reasoning_extra or {}) + # owns_reasoning_policy(), not "the hook was overridden": an unrelated profile + # override (e.g. one that only sets a top-level field) must not implicitly + # suppress the generic reasoning fallback below (providers.base.ProviderProfile + # docstring). + handles_reasoning = bool(provider_profile.owns_reasoning_policy(supports_reasoning=_supports_reasoning)) + if not handles_reasoning and not is_lmstudio and agent._supports_reasoning_extra_body(): extra_body["reasoning"] = agent.reasoning_config if agent.reasoning_config is not None else {"enabled": True, "effort": "medium"} if "nousresearch" in agent._base_url_lower: from agent.portal_tags import nous_portal_tags @@ -2029,14 +2180,14 @@ def _iteration_summary_chat_kwargs(agent, api_messages: list) -> dict: summary_kwargs.update(agent._max_tokens_param(agent.max_tokens)) if lm_reasoning_effort is not None: summary_kwargs["reasoning_effort"] = lm_reasoning_effort + if top_level_from_profile: + summary_kwargs.update(top_level_from_profile) # Merge the profile's canonical body even when routing is unset (e.g. required Portal tags). provider_preferences = _provider_preferences_for_agent(agent) profile_extra_body = {} - with contextlib.suppress(Exception): - from providers import get_provider_profile - provider_profile = get_provider_profile(agent.provider) - if provider_profile is not None: + if provider_profile is not None: + with contextlib.suppress(Exception): profile_extra_body = provider_profile.build_extra_body( session_id=getattr(agent, "session_id", None), provider_preferences=provider_preferences or None, model=agent.model, base_url=agent.base_url, reasoning_config=agent.reasoning_config) @@ -2068,7 +2219,8 @@ def _codex_summary_attempt(agent, api_messages: list, api_request_id: str): def _attempt(retry_count: int) -> str: codex_kwargs = agent._build_api_kwargs(api_messages) codex_kwargs.pop("tools", None) - return _summary_text(agent, agent._run_codex_stream(codex_kwargs)) + response = _dispatch_provider_request(agent, codex_kwargs, agent._run_codex_stream) + return _summary_text(agent, response) return _attempt @@ -2079,7 +2231,10 @@ def _attempt(retry_count: int) -> str: reasoning_config=agent.reasoning_config, is_oauth=agent._is_anthropic_oauth, preserve_dots=agent._anthropic_preserve_dots(), base_url=getattr(agent, "_anthropic_base_url", None)) ant_kw = _merge_nous_portal_messages_extra_body(agent, ant_kw) - response = _managed_summary_call(agent, api_request_id, ant_kw, agent._anthropic_messages_create, retry_count=retry_count) + response = _managed_summary_call( + agent, api_request_id, ant_kw, + lambda request: _dispatch_provider_request(agent, request, agent._anthropic_messages_create), + retry_count=retry_count) return _summary_text(agent, response, strip_tool_prefix=agent._is_anthropic_oauth) return _attempt @@ -2090,7 +2245,10 @@ def _chat_summary_attempt(agent, api_messages: list, api_request_id: str): def _attempt(retry_count: int) -> str: summary_client = agent._ensure_primary_openai_client(reason="iteration_limit_summary_retry" if retry_count else "iteration_limit_summary") response = _managed_summary_call( - agent, api_request_id, summary_kwargs, lambda request: summary_client.chat.completions.create(**request), retry_count=retry_count) + agent, api_request_id, summary_kwargs, + lambda request: _dispatch_provider_request( + agent, request, lambda authorized: summary_client.chat.completions.create(**authorized)), + retry_count=retry_count) return _summary_text(agent, response) return _attempt @@ -2118,6 +2276,13 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: from agent.context_compressor import MAX_ITERATIONS_SUMMARY_REQUEST append_message(messages, {"role": "user", "content": MAX_ITERATIONS_SUMMARY_REQUEST}) + # The physical summary call now goes through the same egress-firewall dispatch as every + # other provider request (previously bypassed it entirely), which requires + # agent._current_api_request_id to be a truthy request identity; stamp this call's id and + # restore whatever the surrounding turn had, since this runs outside the normal per-iteration + # dispatch that would otherwise set it. + prior_api_request_id = getattr(agent, "_current_api_request_id", "") + agent._current_api_request_id = summary_api_request_id try: api_messages = _iteration_summary_api_messages(agent, messages) build_attempt = _SUMMARY_ATTEMPT_BUILDERS.get(agent.api_mode, _chat_summary_attempt) @@ -2141,6 +2306,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: logger.warning("Failed to get summary response: %s", e) final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" finally: + agent._current_api_request_id = prior_api_request_id from agent import relay_llm relay_llm.complete_logical_call(summary_api_request_id, outcome=summary_call_outcome) diff --git a/agent/codex_responses_adapter.py b/agent/codex_responses_adapter.py index 0dadad59b95a6..63a1e6eea8c38 100644 --- a/agent/codex_responses_adapter.py +++ b/agent/codex_responses_adapter.py @@ -778,6 +778,7 @@ def _preflight_tool(tool: Any, idx: int) -> Dict[str, Any]: _PREFLIGHT_ALLOWED_KEYS = { "model", "instructions", "input", "tools", "store", "extra_headers", "extra_body", + "_hermes_source_provenance", *(key for key, _, _ in _PREFLIGHT_OPTIONAL_FIELDS), } @@ -809,6 +810,8 @@ def _preflight_codex_api_kwargs( normalized: Dict[str, Any] = { "model": model.strip(), "instructions": instructions, "input": input_items, "store": False, } + if "_hermes_source_provenance" in api_kwargs: + normalized["_hermes_source_provenance"] = api_kwargs["_hermes_source_provenance"] tools = api_kwargs.get("tools") if tools is not None: if not isinstance(tools, list): diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index e58638cca8094..d0cd0c0f6fb44 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -809,6 +809,9 @@ def _sanitize_consumer_codex_request(agent: Any, request: dict[str, Any]) -> dic middleware / ``request_overrides``): a late ``prompt_cache_retention``, top-level or nested in ``extra_body``, would otherwise HTTP 400 a valid follow-up.""" sanitized = dict(request) + # Internal-only sidecar (egress firewall's post-hoc trust proof for the NEXT request): + # carried through preflight for the runtime to read, never sent to any provider wire. + sanitized.pop("_hermes_source_provenance", None) # getattr: run_codex_stream is also driven with stand-in agents carrying only the attrs a path needs. backend_predicate = getattr(agent, "_is_codex_backend", None) if not (callable(backend_predicate) and bool(backend_predicate())): diff --git a/agent/coding_context.py b/agent/coding_context.py index 178d71eeb9dde..a99bf5f2e6cc9 100644 --- a/agent/coding_context.py +++ b/agent/coding_context.py @@ -195,6 +195,50 @@ def _coding_mode(config: Optional[dict[str, Any]]) -> str: return _MODE_ALIASES.get(str(raw).strip().lower(), "auto") +def guarded_prompt_enabled( + *, + platform: Optional[str] = None, + cwd: Optional[str | Path] = None, + provider: Optional[str] = None, + model: Optional[str] = None, + config: Optional[dict[str, Any]] = None, +) -> bool: + """Return whether the explicitly opt-in guarded coding prompt applies.""" + if config is None: + try: + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() + except Exception: + return False + agent_cfg = (config or {}).get("agent", {}) or {} + if not isinstance(agent_cfg, dict) or _coding_mode(config) != "focus": + return False + raw = agent_cfg.get("guarded_prompt_mode") + if not isinstance(raw, dict) or raw.get("enabled") is not True: + return False + routes = raw.get("routes") + if not isinstance(routes, (list, tuple)): + return False + route_keys = { + ( + str(route.get("provider") or "").strip().lower(), + str(route.get("model") or "").strip().lower(), + ) + for route in routes + if isinstance(route, dict) + } + pair = (str(provider or "").strip().lower(), str(model or "").strip().lower()) + return bool( + pair[0] + and pair[1] + and pair in route_keys + and resolve_runtime_mode( + platform=platform, cwd=cwd, config=config, model=model + ).is_coding + ) + + def _resolve_cwd(cwd: Optional[str | Path]) -> Path: if cwd: return Path(cwd).expanduser() diff --git a/agent/context_compressor.py b/agent/context_compressor.py index ac32a6fc18b10..e5ccc0074546f 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -2639,9 +2639,11 @@ def _truncate_tool_call_args_at(result: List[Dict[str, Any]], idx: int) -> bool: def _demote_tool_result_at( result: List[Dict[str, Any]], idx: int, call_id_to_tool: Dict[str, tuple[str, str]], min_prune_chars: int, protected_skills: Optional[set[str]] = None, + pressure: bool = False, ) -> bool: """Replace the tool result at ``idx`` with a 1-line summary; True if modified. - ``protected_skills`` (lower-cased) spares matching skill_view bodies; None (pressure pass) overrides the guard.""" + ``protected_skills`` (lower-cased) spares matching skill_view bodies; pressure demotion still retains the + current worker assignment while overriding the skill guard.""" msg = result[idx] if msg.get("role") != "tool": return False @@ -2663,7 +2665,14 @@ def _demote_tool_result_at( _skill = _json_dict(tool_args).get("name", "") if isinstance(_skill, str) and _skill.lower() in protected_skills: return False - result[idx] = {**msg, "content": _summarize_tool_result(tool_name, tool_args, content)} + from agent.context_compressor_kanban import newest_assignment_summary + + summary = newest_assignment_summary(result, idx, call_id_to_tool) + if summary is None: + summary = _summarize_tool_result(tool_name, tool_args, content) + if summary == content: + return False + result[idx] = {**msg, "content": summary} return True def _pressure_demote_tail( @@ -2685,7 +2694,7 @@ def _protected_region_tokens() -> int: def _shrink_at(i: int) -> None: # Each helper no-ops on the other role, so both may run unconditionally. nonlocal demoted, pressure_hits - if self._demote_tool_result_at(result, i, call_id_to_tool, min_prune_chars): + if self._demote_tool_result_at(result, i, call_id_to_tool, min_prune_chars, pressure=True): demoted += 1 pressure_hits += 1 if self._truncate_tool_call_args_at(result, i): @@ -2705,7 +2714,7 @@ def _shrink_at(i: int) -> None: # Last resort: the newest body alone may exceed the soft budget; summarize it. if ( last_tool_idx is not None and last_tool_idx >= prune_boundary and _protected_region_tokens() > soft_ceiling - ) and self._demote_tool_result_at(result, last_tool_idx, call_id_to_tool, min_prune_chars): + ) and self._demote_tool_result_at(result, last_tool_idx, call_id_to_tool, min_prune_chars, pressure=True): demoted += 1 pressure_hits += 1 if pressure_hits and not self.quiet_mode: @@ -3073,10 +3082,55 @@ def _build_static_fallback_summary( summary = _reinject_pruned_skill_markers(summary, _pruned_names) return self._augment_summary_lean(summary, turns_to_summarize) + @staticmethod + def _current_assignment_summary( + messages: List[Dict[str, Any]], start: int, end: int, + ) -> Optional[str]: + """Return the newest current-task Kanban projection inside a soon-to-be-dropped window.""" + from agent.context_compressor_kanban import assignment_summary_from_handoff, newest_assignment_summary + + call_id_to_tool = _tool_calls_by_id(messages) + for index in range(min(end, len(messages)) - 1, max(0, start) - 1, -1): + summary = newest_assignment_summary(messages, index, call_id_to_tool) + if summary is not None: + return summary + message = messages[index] + if message.get(COMPRESSED_SUMMARY_METADATA_KEY) or ContextCompressor._is_context_summary_message(message): + summary = assignment_summary_from_handoff( + _content_text_for_contains(message.get("content")), + ) + if summary is not None: + return summary + return None + + @staticmethod + def _append_current_assignment_summary(summary: str, assignment_summary: str) -> str: + """Carry a bounded current-task projection in the deterministic handoff.""" + assignment_summary = _redact_compaction_text(assignment_summary) + if assignment_summary in summary: + return summary + return f"{summary.rstrip()}\n\n[CURRENT KANBAN ASSIGNMENT]\n{assignment_summary}" + def _demote_stale_tail_tools(self, messages: List[Dict[str, Any]], tail_start: int) -> List[Dict[str, Any]]: """Lean mode: demote tail tool results older than the newest ``_LEAN_TAIL_KEEP_TOOL_ROUNDS`` rounds to - recovery stubs; skill-marker rows untouched. New list (untouched rows shared, demoted copied).""" + recovery stubs; the newest current-task Kanban projection is also protected. Skill-marker rows are + untouched. New list (untouched rows shared, demoted copied).""" session_id = getattr(self, "_session_id", "") or "" + call_id_to_tool = _tool_calls_by_id(messages) + from agent.context_compressor_kanban import newest_assignment_summary + + current_assignment_idx = next( + ( + i + for i in range(len(messages) - 1, tail_start - 1, -1) + if newest_assignment_summary(messages, i, call_id_to_tool) is not None + ), + None, + ) + current_assignment_summary = ( + newest_assignment_summary(messages, current_assignment_idx, call_id_to_tool) + if current_assignment_idx is not None else None + ) rounds_seen = 0 protected: set[int] = set() prev_idx = None @@ -3091,7 +3145,15 @@ def _demote_stale_tail_tools(self, messages: List[Dict[str, Any]], tail_start: i for i in range(tail_start, len(messages)): msg = messages[i] content = msg.get("content") - if msg.get("role") != "tool" or i in protected or not isinstance(content, str): + if ( + msg.get("role") != "tool" + or i in protected + or not isinstance(content, str) + ): + continue + if i == current_assignment_idx: + if current_assignment_summary is not None: + result[i] = _rewritten(msg, current_assignment_summary) continue if len(content) < _LEAN_TAIL_DEMOTE_MIN_CHARS or SKILL_PRUNED_MARKER_PREFIX in content or _is_summary_stub(content): continue @@ -4640,6 +4702,11 @@ def compress( if getattr(self, "tail_mode", "lean") == "lean": messages = self._demote_stale_tail_tools(messages, compress_end) scan = self._scan_window_handoffs(messages, compress_start, compress_end, turns_to_summarize) + # Scan the actual handoff-expanded window (scan.tail_start may sit past + # compress_end when a later handoff was consumed), not just the initial + # [compress_start, compress_end) slice -- otherwise a stale in-window + # match shadows a newer assignment carried by that later-consumed handoff. + current_assignment_summary = self._current_assignment_summary(messages, compress_start, scan.tail_start) turns_to_summarize = scan.turns_to_summarize self._record_compression_regions( head_messages=messages[:compress_start], middle_messages=turns_to_summarize, tail_messages=messages[compress_end:], @@ -4673,6 +4740,9 @@ def compress( summary = self._fallback_summary_for_window( telemetry, turns_to_summarize, compress_end - compress_start, feasibility_skip, ) + if current_assignment_summary: + summary = self._append_current_assignment_summary(summary, current_assignment_summary) + self._previous_summary = self._strip_summary_prefix(summary) # Phase 4: Assemble compressed message list compressed = self._assemble_compressed(messages, compress_start, compress_end, scan, summary) return self._finalize_compressed(compressed, messages, n_messages) diff --git a/agent/context_compressor_kanban.py b/agent/context_compressor_kanban.py new file mode 100644 index 0000000000000..347296b87758f --- /dev/null +++ b/agent/context_compressor_kanban.py @@ -0,0 +1,102 @@ +"""Keep the newest bounded Kanban assignment through tool-result compression.""" +import json +import os + + +def _assignment(message, calls): + name, arguments = calls.get(message.get("tool_call_id", ""), (None, "")) + if message.get("role") != "tool" or name != "kanban_show": + return None + try: + payload = json.loads(message.get("content", "")) + arguments = json.loads(arguments) + except (TypeError, ValueError): + return None + if not isinstance(payload, dict) or not isinstance(arguments, dict): + return None + task = payload.get("task") + worker_task_id = os.environ.get("HERMES_KANBAN_TASK") + task_id = arguments.get("task_id") or worker_task_id + if worker_task_id != task_id: + return None + if not isinstance(task, dict) or not isinstance(task_id, str) or not task_id or len(task_id) > 128 or task.get("id") != task_id: + return None + if not isinstance(task.get("title"), str) or not isinstance(task.get("body"), str): + return None + return task_id, payload + + +def _bounded(text, budget): + encoded = text.encode("utf-8", errors="replace") + if len(encoded) <= budget: + return encoded.decode("utf-8") + suffix = "\n" + return encoded[:budget - len(suffix)].decode("utf-8", errors="ignore") + suffix + + +def newest_assignment_summary(messages, index, calls): + current = _assignment(messages[index], calls) + if current is None: + return None + task_id, payload = current + if any((later := _assignment(message, calls)) is not None and later[0] == task_id + for message in messages[index + 1:]): + return None + task = payload["task"] + projected = {"id": task_id, "title": _bounded(task["title"], 1024), + "body": _bounded(task["body"], 8 * 1024)} + for key in ("status", "workspace_access"): + if isinstance(task.get(key), str): + projected[key] = _bounded(task[key], 128) + summary = {"task": projected} + spec = payload.get("protected_task_spec") + # Preserve an existing producer contract; never promote ordinary task text + # into a protected egress grant or bring back historical board material. + if (isinstance(spec, dict) and spec.get("version") == "v1" + and isinstance(spec.get("title"), str) and isinstance(spec.get("body"), str)): + summary["protected_task_spec"] = { + "version": "v1", "title": _bounded(spec["title"], 1024), + "body": _bounded(spec["body"], 8 * 1024), + } + return json.dumps(summary, ensure_ascii=False) + + +def assignment_summary_from_handoff(content): + """Extract the bounded current-task projection from a prior compression handoff.""" + marker = "[CURRENT KANBAN ASSIGNMENT]" + if not isinstance(content, str): + return None + marker_index = content.rfind(marker) + if marker_index < 0: + return None + try: + payload, _ = json.JSONDecoder().raw_decode(content[marker_index + len(marker):].lstrip()) + except (TypeError, ValueError): + return None + if not isinstance(payload, dict): + return None + task = payload.get("task") + worker_task_id = os.environ.get("HERMES_KANBAN_TASK") + if ( + not isinstance(task, dict) + or not isinstance(worker_task_id, str) + or not worker_task_id + or task.get("id") != worker_task_id + or not isinstance(task.get("title"), str) + or not isinstance(task.get("body"), str) + ): + return None + projected = {"id": worker_task_id, "title": _bounded(task["title"], 1024), + "body": _bounded(task["body"], 8 * 1024)} + for key in ("status", "workspace_access"): + if isinstance(task.get(key), str): + projected[key] = _bounded(task[key], 128) + summary = {"task": projected} + spec = payload.get("protected_task_spec") + if (isinstance(spec, dict) and spec.get("version") == "v1" + and isinstance(spec.get("title"), str) and isinstance(spec.get("body"), str)): + summary["protected_task_spec"] = { + "version": "v1", "title": _bounded(spec["title"], 1024), + "body": _bounded(spec["body"], 8 * 1024), + } + return json.dumps(summary, ensure_ascii=False) diff --git a/agent/context_references.py b/agent/context_references.py index 5911ff7f848a7..d7cc407d51e6d 100644 --- a/agent/context_references.py +++ b/agent/context_references.py @@ -168,10 +168,17 @@ def parse_context_references(message: str) -> list[ContextReference]: def preprocess_context_references( message: str, *, cwd: str | Path, context_length: int, url_fetcher: UrlFetcher = None, allowed_root: str | Path | None = None, + source_provenance_registry=None, + session_id: str | None = None, + turn_id: str | None = None, + request_id: str | None = None, + policy_digest: str | None = None, ) -> ContextReferenceResult: """Sync wrapper; safe both without a loop (CLI) and inside a running loop (gateway).""" coro = preprocess_context_references_async( - message, cwd=cwd, context_length=context_length, url_fetcher=url_fetcher, allowed_root=allowed_root + message, cwd=cwd, context_length=context_length, url_fetcher=url_fetcher, allowed_root=allowed_root, + source_provenance_registry=source_provenance_registry, session_id=session_id, turn_id=turn_id, + request_id=request_id, policy_digest=policy_digest, ) try: asyncio.get_running_loop() @@ -185,6 +192,11 @@ def preprocess_context_references( async def preprocess_context_references_async( message: str, *, cwd: str | Path, context_length: int, url_fetcher: UrlFetcher = None, allowed_root: str | Path | None = None, + source_provenance_registry=None, + session_id: str | None = None, + turn_id: str | None = None, + request_id: str | None = None, + policy_digest: str | None = None, ) -> ContextReferenceResult: refs = parse_context_references(message) if not refs: @@ -199,7 +211,8 @@ async def preprocess_context_references_async( soft_limit = max(1, int(context_length * 0.25)) tasks = ( _expand_reference(ref, cwd_path, url_fetcher=url_fetcher, allowed_root=allowed_root_path, - max_inline_tokens=hard_limit) + max_inline_tokens=hard_limit, source_provenance_registry=source_provenance_registry, + session_id=session_id, turn_id=turn_id, request_id=request_id, policy_digest=policy_digest) for ref in refs ) expanded = await asyncio.gather(*tasks) @@ -241,10 +254,16 @@ async def preprocess_context_references_async( async def _expand_reference( ref: ContextReference, cwd: Path, *, url_fetcher: UrlFetcher = None, allowed_root: Path | None = None, max_inline_tokens: int | None = None, + source_provenance_registry=None, session_id: str | None = None, turn_id: str | None = None, + request_id: str | None = None, policy_digest: str | None = None, ) -> Expansion: try: if ref.kind in ("file", "folder"): - return _expand_path_reference(ref, cwd, allowed_root=allowed_root, max_inline_tokens=max_inline_tokens) + return _expand_path_reference( + ref, cwd, allowed_root=allowed_root, max_inline_tokens=max_inline_tokens, + source_provenance_registry=source_provenance_registry, session_id=session_id, turn_id=turn_id, + request_id=request_id, policy_digest=policy_digest, + ) if ref.kind in _GIT_REFERENCE_ARGS: git_args = _GIT_REFERENCE_ARGS[ref.kind](ref) return _expand_git_reference(ref, cwd, git_args, "git " + " ".join(git_args)) @@ -267,9 +286,15 @@ async def _expand_reference( def _expand_path_reference(ref: ContextReference, cwd: Path, *, allowed_root: Path | None = None, - max_inline_tokens: int | None = None) -> Expansion: + max_inline_tokens: int | None = None, + source_provenance_registry=None, session_id: str | None = None, + turn_id: str | None = None, request_id: str | None = None, + policy_digest: str | None = None) -> Expansion: """``@file:`` / ``@folder:``: resolve, allow-check, then inline text / binary stub / listing.""" is_folder = ref.kind == "folder" + # The unresolved spelling: _resolve_path()'s .resolve() call follows/erases any symlink + # component, so this is what a provenance grant must inspect to catch one (below). + unresolved_path = cwd / Path(os.path.expanduser(ref.target)) path = _resolve_path(cwd, ref.target, allowed_root=allowed_root) _ensure_reference_path_allowed(path) if not path.exists(): @@ -285,7 +310,29 @@ def _expand_path_reference(ref: ContextReference, cwd: Path, *, allowed_root: Pa return None, _binary_reference_block(ref, path) text = path.read_text(encoding="utf-8") if ref.line_start is not None: - text = "\n".join(text.splitlines()[max(ref.line_start - 1, 0):ref.line_end or ref.line_start]) + line_end = ref.line_end or ref.line_start + text = "\n".join(text.splitlines()[max(ref.line_start - 1, 0):line_end]) + # An exact bounded slice can carry trusted provenance for the egress boundary — but only + # when the caller supplied a full request identity, and never through a path whose + # ancestors include a symlink (an attacker-controlled indirection could point the same + # spelling at different bytes between the read and a later trust check). + if ( + source_provenance_registry is not None + and all(isinstance(v, str) and v for v in (session_id, turn_id, request_id, policy_digest)) + ): + from agent.source_provenance import SourceProvenanceError + try: + # issue_file_slice byte-compares against its own independent re-read, which keeps + # each line's original terminator — must match exactly, not the display text above + # (splitlines() + "\n".join() drops a trailing newline the source file still has). + raw_lines = path.read_bytes().splitlines(keepends=True) + raw_slice = b"".join(raw_lines[max(ref.line_start - 1, 0):line_end]) + source_provenance_registry.issue_file_slice( + path=unresolved_path, line_start=ref.line_start, line_end=line_end, content=raw_slice, + session_id=session_id, turn_id=turn_id, request_id=request_id, policy_digest=policy_digest, + ) + except SourceProvenanceError as exc: + return f"{ref.raw}: source provenance grant declined ({exc})", None lang = _FENCE_LANGUAGES.get(path.suffix.lower(), "") text_tokens = estimate_tokens_rough(text) # Check BEFORE building the fenced block: an oversized file is not going to be diff --git a/agent/conversation_worktree.py b/agent/conversation_worktree.py new file mode 100644 index 0000000000000..ca8b4894c2388 --- /dev/null +++ b/agent/conversation_worktree.py @@ -0,0 +1,1572 @@ +"""Durable, fail-closed Git worktrees for interactive conversation roots.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import hashlib +import json +import logging +import os +from pathlib import Path +import re +import subprocess +import threading +import time +from typing import Iterator +import uuid + +from agent.conversation_worktree_policy import ConversationWorktreePolicy +from hermes_cli.worktree_environment import bootstrap_worktree_environments +from hermes_cli._subprocess_compat import ( + IS_WINDOWS, + kill_process_tree, + noninteractive_git_env, + windows_hide_flags, +) +from hermes_state import SessionDB +from hermes_state_worktrees import ( + ConversationWorktreeConflict, + ConversationWorktreeRecord, +) + + +logger = logging.getLogger(__name__) + +_REPOSITORY_THREAD_LOCKS: dict[str, threading.Lock] = {} +_REPOSITORY_THREAD_LOCKS_GUARD = threading.Lock() +_OWNER_MARKER = "hermes-conversation-owner-v1" +_COMMON_OWNER_CLAIMS_DIR = "hermes-conversation-owner-claims-v1" +_LEASE_MESSAGE_LIMIT = 300 + + +class ConversationWorktreeError(RuntimeError): + """A conversation worktree could not be safely created or reused.""" + + def __init__(self, message: str, *, phase: str) -> None: + super().__init__(message) + self.phase = phase + + +@dataclass(frozen=True) +class ConversationWorktreeBinding: + """A validated, task-owned working directory for one conversation root.""" + + root_session_id: str + path: Path + branch: str + base_commit: str + repo_common_dir: Path + + +@dataclass(frozen=True) +class CleanupVerdict: + """Complete fail-closed status for one explicit cleanup request.""" + + allowed: bool + reasons: tuple[str, ...] + + +@dataclass(frozen=True) +class CleanupResult: + """Result of an explicit cleanup attempt and the verdict that governed it.""" + + removed: bool + verdict: CleanupVerdict + failure_phase: str | None = None + failure_message: str | None = None + + +@dataclass +class ConversationRootLease: + """Mandatory cross-process ownership lease for one conversation root.""" + + lease_id: str + root_session_id: str + state_path: Path + lock_path: Path + released: bool = False + + def release(self) -> None: + if self.released: + return + try: + with _lease_file_lock(self.lock_path, timeout=3.0): + entries, valid = _read_root_leases(self.state_path) + if not valid: + raise ConversationWorktreeError( + "conversation root lease registry is unavailable", + phase="lease", + ) + _write_root_leases( + self.state_path, + [e for e in entries if e.get("lease_id") != self.lease_id], + ) + except ConversationWorktreeError: + raise + except Exception as exc: + raise ConversationWorktreeError( + "conversation root lease registry is unavailable", phase="lease" + ) from exc + self.released = True + + +def _root_lease_paths(repo_common_dir: Path, root_session_id: str) -> tuple[Path, Path]: + digest = hashlib.sha256(root_session_id.encode("utf-8")).hexdigest()[:24] + return ( + repo_common_dir / f"hermes-conversation-root-{digest}.leases.json", + repo_common_dir / f"hermes-conversation-root-{digest}.leases.lock", + ) + + +@contextmanager +def _lease_file_lock(path: Path, *, timeout: float) -> Iterator[None]: + deadline = time.monotonic() + timeout + handle = None + locked = False + thread_lock = ConversationWorktreeManager._thread_lock(path) + if not thread_lock.acquire(timeout=timeout): + raise OSError("conversation root lease lock timed out") + try: + path.parent.mkdir(parents=True, exist_ok=True) + handle = path.open("a+b") + handle.seek(0) + handle.write(b"0") + handle.flush() + while True: + try: + if os.name == "nt": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + locked = True + break + except OSError: + if time.monotonic() >= deadline: + raise OSError("conversation root lease lock timed out") + time.sleep(0.05) + yield + finally: + if handle is not None: + if locked: + try: + if os.name == "nt": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: + logger.warning("conversation_worktree.lease_lock_release_failed") + handle.close() + thread_lock.release() + + +def _read_root_leases(path: Path) -> tuple[list[dict[str, object]], bool]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return [], True + except Exception: + return [], False + entries = data.get("entries") if isinstance(data, dict) else None + if not isinstance(entries, list) or not all(isinstance(e, dict) for e in entries): + return [], False + return list(entries), True + + +def _write_root_leases(path: Path, entries: list[dict[str, object]]) -> None: + tmp = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") + with tmp.open("w", encoding="utf-8") as handle: + json.dump({"entries": entries}, handle, sort_keys=True) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + + +def _process_liveness(entry: dict[str, object]) -> str: + try: + pid = int(entry.get("pid") or 0) + except (TypeError, ValueError): + return "unknown" + if pid <= 0: + return "unknown" + try: + import psutil + + if not psutil.pid_exists(pid): + return "dead" + except Exception: + return "unknown" + expected = entry.get("process_start_time") + if expected is None: + return "active" + try: + from hermes_cli.active_sessions import _process_start_time + + current = _process_start_time(pid) + if current is None: + return "unknown" + return "active" if abs(float(expected) - current) < 0.001 else "dead" + except Exception: + return "unknown" + + +def acquire_conversation_root_lease( + *, + root_session_id: str, + worktree_path: Path, + repo_common_dir: Path, + surface: str, +) -> ConversationRootLease: + """Acquire mandatory root liveness independent of concurrency limits.""" + state_path, lock_path = _root_lease_paths(repo_common_dir, root_session_id) + lease_id = uuid.uuid4().hex + try: + with _lease_file_lock(lock_path, timeout=3.0): + entries, valid = _read_root_leases(state_path) + if not valid: + raise ConversationWorktreeError( + "conversation root lease registry is unavailable", phase="lease" + ) + kept = [e for e in entries if _process_liveness(e) != "dead"] + kept.append( + { + "lease_id": lease_id, + "root_session_id": root_session_id, + "worktree_path": str(worktree_path.resolve()), + "repo_common_dir": str(repo_common_dir.resolve()), + "surface": str(surface), + "pid": os.getpid(), + "process_start_time": __import__( + "hermes_cli.active_sessions", fromlist=["_process_start_time"] + )._process_start_time(os.getpid()), + "started_at": time.time(), + } + ) + _write_root_leases(state_path, kept) + except ConversationWorktreeError: + raise + except Exception as exc: + raise ConversationWorktreeError( + "conversation root lease registry is unavailable", phase="lease" + ) from exc + return ConversationRootLease( + lease_id=lease_id, + root_session_id=root_session_id, + state_path=state_path, + lock_path=lock_path, + ) + + +def _common_owner_claim_path(repo_common_dir: Path, worktree_path: Path) -> Path: + """Return the source-Git durable claim location for one exact worktree path.""" + digest = hashlib.sha256(str(worktree_path.resolve()).encode("utf-8")).hexdigest() + return repo_common_dir / _COMMON_OWNER_CLAIMS_DIR / f"{digest}.json" + + +def _owner_claim_matches( + data: object, *, worktree_path: Path, repo_common_dir: Path +) -> bool: + return ( + isinstance(data, dict) + and data.get("owner") == "conversation-worktree-manager" + and data.get("worktree_path") == str(worktree_path.resolve()) + and data.get("repo_common_dir") == str(repo_common_dir.resolve()) + and isinstance(data.get("root_session_id"), str) + and bool(data["root_session_id"]) + ) + + +def _owner_claim_matches_record( + data: object, *, record: ConversationWorktreeRecord +) -> bool: + """Require ownership evidence for this exact durable conversation root.""" + return ( + _owner_claim_matches( + data, + worktree_path=Path(record.worktree_path), + repo_common_dir=Path(record.repo_common_dir), + ) + and data.get("root_session_id") == record.root_session_id + ) + + +def conversation_worktree_ownership_verdict(path: Path) -> bool | None: + """Return manager ownership, absence, or an unverified ownership state. + + The common-repository claim is written before ``git worktree add``. It + closes the crash window where Git has created the worktree but the + worktree-local marker cannot yet be persisted. Any malformed or + unreadable ownership evidence is deliberately ``None`` so every GC caller + keeps the tree rather than guessing that it is safe to remove. + """ + try: + result = subprocess.run( + [ + "git", + "-C", + str(path), + "rev-parse", + "--path-format=absolute", + "--git-common-dir", + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0 or not result.stdout.strip(): + return None + repo_common_dir = Path(result.stdout.strip()).resolve() + + try: + marker_result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "--git-path", _OWNER_MARKER], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if marker_result.returncode != 0 or not marker_result.stdout.strip(): + return None + marker = Path(marker_result.stdout.strip()) + if not marker.is_absolute(): + marker = path / marker + if marker.exists(): + try: + if _owner_claim_matches( + json.loads(marker.read_text(encoding="utf-8")), + worktree_path=path, + repo_common_dir=repo_common_dir, + ): + return True + return None + except Exception: + return None + + common_claim = _common_owner_claim_path(repo_common_dir, path) + if not common_claim.exists(): + return False + try: + if _owner_claim_matches( + json.loads(common_claim.read_text(encoding="utf-8")), + worktree_path=path, + repo_common_dir=repo_common_dir, + ): + return True + except Exception: + pass + return None + + +def conversation_worktree_is_manager_owned(path: Path) -> bool | None: + """Return ownership status; ``None`` means ownership cannot be verified.""" + return conversation_worktree_ownership_verdict(path) + + +def _write_owner_claim(path: Path, payload: dict[str, str]) -> None: + """Atomically write ownership evidence without platform-specific renames.""" + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") + with tmp.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, sort_keys=True) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + except OSError as exc: + raise ConversationWorktreeError( + "conversation worktree ownership claim is unavailable", phase="create" + ) from exc + + +@contextmanager +def conversation_worktree_reclaim_guard( + repo_root: Path, path: Path +) -> Iterator[bool | None]: + """Hold the manager repository lock while rechecking durable ownership.""" + try: + result = subprocess.run( + [ + "git", + "-C", + str(repo_root), + "rev-parse", + "--path-format=absolute", + "--git-common-dir", + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + yield None + return + common_dir = Path(result.stdout.strip()).resolve() + with _lease_file_lock( + common_dir / "hermes-conversation-worktree.lock", timeout=5.0 + ): + yield conversation_worktree_is_manager_owned(path) + except Exception: + yield None + + +class ConversationWorktreeManager: + """Own the Git lifecycle for interactive root-session worktrees only.""" + + def __init__(self, policy: ConversationWorktreePolicy, db: SessionDB) -> None: + self._policy = policy + self._db = db + + def bind_new_root_session( + self, root_session_id: str, *, conversation_kind: str + ) -> ConversationWorktreeBinding | None: + """Create or recover an interactive root binding without touching source work. + + Task/delegated callers deliberately bypass this manager: they retain their + existing worktree ownership instead of being converted into conversation + roots. + """ + self._event("conversation_worktree.policy", root_session_id=root_session_id) + if conversation_kind == "task": + return None + if conversation_kind != "interactive": + raise ConversationWorktreeError( + "conversation kind must be 'interactive' or 'task'", phase="policy" + ) + if not self._policy.enabled: + return None + if not root_session_id: + raise ConversationWorktreeError( + "root session id must be non-empty", phase="identity" + ) + + existing = self._db.get_conversation_worktree(root_session_id) + source, source_common_dir = self._source_repository_identity() + path, branch = self._expected_identity(root_session_id) + if self._is_within(path, source): + raise ConversationWorktreeError( + "worktree_root must not create conversation worktrees inside source_worktree", + phase="policy", + ) + record = existing + try: + with self._repository_lock(source_common_dir): + record = self._db.get_conversation_worktree(root_session_id) or record + if record is not None and record.state == "ready": + return self._resolve_ready_binding_locked( + source, + source_common_dir, + record, + path=path, + branch=branch, + ) + self._validate_worktree_root_ownership( + source, + source_common_dir, + expected_path=path, + existing=record, + ) + if record is None: + base_commit = self._git_stdout( + source, ["rev-parse", "HEAD"], "identity" + ) + try: + record = self._db.claim_conversation_worktree( + root_session_id=root_session_id, + worktree_path=str(path), + branch=branch, + base_commit=base_commit, + repo_common_dir=str(source_common_dir), + ) + except ConversationWorktreeConflict as exc: + record = self._db.get_conversation_worktree(root_session_id) + if record is None: + raise ConversationWorktreeError( + "conversation worktree identity claim conflicted", + phase="identity", + ) from exc + self._validate_record_identity( + record, + path=path, + branch=branch, + repo_common_dir=source_common_dir, + ) + self._prepare_worktree(source, record) + self._ensure_git_worktree_locked(source, record) + + # Bootstrap may be slow, network-bound, or intentionally interactive + # at the project level. It must never monopolize the repository-wide + # Git metadata lock: that lock protects only claim/create/validation. + # A root-specific lock serializes bootstrap/readiness for retries of + # this one root without blocking different conversation roots. + with self._root_lock(source_common_dir, root_session_id): + record = self._db.get_conversation_worktree(root_session_id) or record + if record.state == "ready": + return self._validated_ready_binding(record) + self._validate_record_identity( + record, + path=path, + branch=branch, + repo_common_dir=source_common_dir, + ) + self._require_recoverable_record(record) + self._validate_new_worktree(record) + self._run_bootstrap(record) + ready = self._db.mark_conversation_worktree_ready(root_session_id) + binding = self._binding_from_record(ready) + self._event("conversation_worktree.ready", root_session_id=root_session_id) + return binding + except ConversationWorktreeError as exc: + self._record_failure(root_session_id, record, exc) + self._event( + "conversation_worktree.failure", + root_session_id=root_session_id, + phase=exc.phase, + ) + raise + + def resolve_existing_session( + self, root_session_id: str + ) -> ConversationWorktreeBinding | None: + """Resolve only an already-ready root binding; never create a new one.""" + record = self._db.get_conversation_worktree(root_session_id) + if record is None: + return None + source, source_common_dir = self._source_repository_identity() + path, branch = self._expected_identity(root_session_id) + if self._is_within(path, source): + raise ConversationWorktreeError( + "worktree_root must not create conversation worktrees inside source_worktree", + phase="policy", + ) + with self._repository_lock(source_common_dir): + record = self._db.get_conversation_worktree(root_session_id) + if record is None: + return None + return self._resolve_ready_binding_locked( + source, + source_common_dir, + record, + path=path, + branch=branch, + ) + + def _resolve_ready_binding_locked( + self, + source: Path, + source_common_dir: Path, + record: ConversationWorktreeRecord, + *, + path: Path, + branch: str, + ) -> ConversationWorktreeBinding: + """Validate and retain one ready binding while holding its repository lock.""" + self._validate_worktree_root_ownership( + source, + source_common_dir, + expected_path=path, + existing=record, + ) + binding = self._validated_ready_binding(record) + self._ensure_git_worktree_locked(source, record) + return binding + + def inspect_cleanup( + self, + root_session_id: str, + *, + active_session_bound: bool = False, + ) -> CleanupVerdict: + """Inspect one exact owned binding without changing Git or ledger state.""" + record = self._db.get_conversation_worktree(root_session_id) + if record is None: + return CleanupVerdict(False, ("unknown",)) + try: + with self._root_lease_liveness(record) as root_liveness: + return self._inspect_cleanup_record( + record, + active_session_bound=active_session_bound, + root_liveness=root_liveness, + ) + except Exception: + return CleanupVerdict(False, ("unknown",)) + + def remove_after_explicit_request( + self, + root_session_id: str, + *, + active_session_bound: bool = False, + ) -> CleanupResult: + """Remove only a re-inspected safe binding after an explicit request.""" + record = self._db.get_conversation_worktree(root_session_id) + if record is None: + verdict = CleanupVerdict(False, ("unknown",)) + return CleanupResult(False, verdict) + + try: + source, source_common_dir = self._source_repository_identity() + with self._repository_lock(source_common_dir): + with self._root_lock(source_common_dir, root_session_id): + current = self._db.get_conversation_worktree(root_session_id) + if current is None: + verdict = CleanupVerdict(False, ("unknown",)) + return CleanupResult(False, verdict) + with self._root_lease_liveness(current) as root_liveness: + verdict = self._inspect_cleanup_record( + current, + active_session_bound=active_session_bound, + root_liveness=root_liveness, + source_identity=(source, source_common_dir), + ) + if not verdict.allowed: + return CleanupResult(False, verdict) + + path = Path(current.worktree_path).resolve() + unlocked = self._run_git( + source, + ["worktree", "unlock", str(path)], + self._policy.create_timeout, + "cleanup", + ) + if unlocked.returncode != 0: + message = self._sanitize_remove_failure(unlocked.stderr) + logger.warning( + "conversation_worktree.remove_failed phase=unlock message=%s", + message, + ) + return CleanupResult( + False, + CleanupVerdict(False, ("remove_failed",)), + failure_phase="unlock", + failure_message=message, + ) + removed = self._run_git( + source, + ["worktree", "remove", str(path)], + self._policy.create_timeout, + "cleanup", + ) + if removed.returncode != 0: + try: + self._ensure_git_worktree_locked(source, current) + except ConversationWorktreeError: + logger.warning( + "conversation_worktree.relock_after_remove_failure_failed", + exc_info=True, + ) + message = self._sanitize_remove_failure(removed.stderr) + logger.warning( + "conversation_worktree.remove_failed phase=remove message=%s", + message, + ) + return CleanupResult( + False, + CleanupVerdict(False, ("remove_failed",)), + failure_phase="remove", + failure_message=message, + ) + if path.exists() or self._listed_worktree(source, path) is not None: + message = "worktree remained after git removal" + logger.warning( + "conversation_worktree.remove_failed phase=verify message=%s", + message, + ) + return CleanupResult( + False, + CleanupVerdict(False, ("remove_failed",)), + failure_phase="verify", + failure_message=message, + ) + + self._db.mark_conversation_worktree_removed(root_session_id) + self._remove_common_owner_claim(current) + self._event( + "conversation_worktree.removed", + root_session_id=root_session_id, + ) + return CleanupResult(True, verdict) + except ConversationWorktreeError: + return CleanupResult(False, CleanupVerdict(False, ("unknown",))) + except Exception: + logger.exception("conversation_worktree.cleanup_failed") + return CleanupResult(False, CleanupVerdict(False, ("unknown",))) + + def _inspect_cleanup_record( + self, + record: ConversationWorktreeRecord, + *, + active_session_bound: bool, + root_liveness: str = "inactive", + source_identity: tuple[Path, Path] | None = None, + ) -> CleanupVerdict: + reasons: list[str] = [] + + def block(reason: str) -> None: + if reason not in reasons: + reasons.append(reason) + + try: + source, source_common_dir = source_identity or self._source_repository_identity() + expected_path, expected_branch = self._expected_identity(record.root_session_id) + if ( + record.state not in {"ready", "retained"} + or Path(record.worktree_path).resolve() != expected_path.resolve() + or record.branch != expected_branch + or Path(record.repo_common_dir).resolve() != source_common_dir.resolve() + or not expected_path.is_dir() + ): + return CleanupVerdict(False, ("mismatched identity",)) + + # A conversation can legitimately rename its branch while preparing or + # merging a PR (mirrors the acceptance in _validated_ready_binding): + # accept that narrow drift only when both independent owner claims + # still bind the exact root, path, and common repository. + renamed_branch_accepted = self._exact_owner_claims_present(record) + + listed = self._listed_worktree(source, expected_path) + if listed != f"refs/heads/{record.branch}": + if not listed or not listed.startswith("refs/heads/") or not renamed_branch_accepted: + return CleanupVerdict(False, ("mismatched identity",)) + + actual_branch = self._git_stdout( + expected_path, ["branch", "--show-current"], "cleanup" + ) + actual_common = Path( + self._git_stdout( + expected_path, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + "cleanup", + ) + ).resolve() + base_ancestor = self._run_git( + expected_path, + ["merge-base", "--is-ancestor", record.base_commit, "HEAD"], + self._policy.create_timeout, + "cleanup", + ) + if ( + (actual_branch != record.branch and not (actual_branch and renamed_branch_accepted)) + or actual_common != source_common_dir.resolve() + or base_ancestor.returncode != 0 + ): + return CleanupVerdict(False, ("mismatched identity",)) + + if active_session_bound: + block("active") + if root_liveness == "active": + block("active") + elif root_liveness != "inactive": + block("unknown") + + status = self._run_git( + expected_path, + ["status", "--porcelain", "--untracked-files=all"], + self._policy.create_timeout, + "cleanup", + ) + if status.returncode != 0: + block("unknown") + elif status.stdout.strip(): + block("dirty") + + if self._git_operation_in_progress(expected_path): + block("in-progress") + + head = self._git_stdout(expected_path, ["rev-parse", "HEAD"], "cleanup") + integration_head = self._git_stdout(source, ["rev-parse", "HEAD"], "cleanup") + integrated = self._run_git( + source, + ["merge-base", "--is-ancestor", head, integration_head], + self._policy.create_timeout, + "cleanup", + ) + if integrated.returncode == 1: + block("unintegrated") + elif integrated.returncode != 0: + block("unknown") + + remotes = self._run_git( + expected_path, + ["remote"], + self._policy.create_timeout, + "cleanup", + ) + if remotes.returncode != 0: + block("unknown") + elif not remotes.stdout.strip(): + block("missing remote evidence") + else: + remote_refs = self._run_git( + expected_path, + [ + "for-each-ref", + "--format=%(refname)", + "--contains", + head, + "refs/remotes", + ], + self._policy.create_timeout, + "cleanup", + ) + if remote_refs.returncode != 0: + block("unknown") + elif not remote_refs.stdout.strip(): + block("unpushed") + except Exception: + block("unknown") + + return CleanupVerdict(not reasons, tuple(reasons)) + + def _listed_worktree(self, source: Path, expected_path: Path) -> str | None: + result = self._run_git( + source, + ["worktree", "list", "--porcelain"], + self._policy.create_timeout, + "cleanup", + ) + if result.returncode != 0: + raise ConversationWorktreeError( + "git worktree inspection failed", phase="cleanup" + ) + path: Path | None = None + branch: str | None = None + for line in [*result.stdout.splitlines(), ""]: + if line.startswith("worktree "): + path = Path(line.removeprefix("worktree ")).resolve() + branch = None + elif line.startswith("branch "): + branch = line.removeprefix("branch ").strip() + elif not line and path is not None: + if path == expected_path.resolve(): + return branch + path = None + branch = None + return None + + def _git_operation_in_progress(self, path: Path) -> bool: + markers = ( + "MERGE_HEAD", + "CHERRY_PICK_HEAD", + "REVERT_HEAD", + "BISECT_LOG", + "rebase-apply", + "rebase-merge", + "sequencer", + "index.lock", + "HEAD.lock", + "packed-refs.lock", + ) + branch = self._git_stdout(path, ["branch", "--show-current"], "cleanup") + if branch: + markers = (*markers, f"refs/heads/{branch}.lock") + for marker in markers: + marker_path = Path( + self._git_stdout(path, ["rev-parse", "--git-path", marker], "cleanup") + ) + if not marker_path.is_absolute(): + marker_path = path / marker_path + if marker_path.exists(): + return True + return False + + @contextmanager + def _root_lease_liveness( + self, record: ConversationWorktreeRecord + ) -> Iterator[str]: + state_path, lock_path = _root_lease_paths( + Path(record.repo_common_dir), record.root_session_id + ) + try: + with _lease_file_lock(lock_path, timeout=self._policy.create_timeout): + entries, valid = _read_root_leases(state_path) + if not valid: + yield "unknown" + return + relevant: list[dict[str, object]] = [] + uncertain = False + for entry in entries: + if ( + entry.get("root_session_id") != record.root_session_id + or Path(str(entry.get("worktree_path") or "")).resolve() + != Path(record.worktree_path).resolve() + or Path(str(entry.get("repo_common_dir") or "")).resolve() + != Path(record.repo_common_dir).resolve() + ): + uncertain = True + continue + state = _process_liveness(entry) + if state == "active": + relevant.append(entry) + elif state == "unknown": + uncertain = True + if len(relevant) != len(entries): + live_or_unknown = [ + entry + for entry in entries + if _process_liveness(entry) != "dead" + ] + if len(live_or_unknown) != len(entries): + _write_root_leases(state_path, live_or_unknown) + yield "unknown" if uncertain else ("active" if relevant else "inactive") + except Exception: + yield "unknown" + + @staticmethod + def _sanitize_remove_failure(stderr: str) -> str: + text = re.sub(r"[\x00-\x1f\x7f]+", " ", str(stderr or "")) + text = re.sub( + r"(?i)\b(token|password|secret|authorization)\s*=\s*\S+", + r"\1=", + text, + ) + text = " ".join(text.split()) or "git worktree remove failed" + return text[:_LEASE_MESSAGE_LIMIT] + + def _source_repository_identity(self) -> tuple[Path, Path]: + source = self._policy.source_worktree + if source is None: + raise ConversationWorktreeError( + "enabled conversation worktree policy has no source_worktree", phase="policy" + ) + source = source.resolve() + if not source.is_dir(): + raise ConversationWorktreeError("source_worktree does not exist", phase="identity") + common = self._git_stdout( + source, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + "identity", + ) + common_dir = Path(common).resolve() + if not common_dir.is_dir(): + raise ConversationWorktreeError( + "source_worktree did not resolve to a usable Git common directory", + phase="identity", + ) + return source, common_dir + + def _validate_worktree_root_ownership( + self, + source: Path, + source_common_dir: Path, + *, + expected_path: Path, + existing: ConversationWorktreeRecord | None, + ) -> None: + """Refuse a configured output root owned by a different repository. + + A worktree directory may sit under a parent repository even when the + configured path itself has not been created yet. Resolve the nearest + existing ancestor and let Git discover its common directory from there; + only a same-common-dir owner is compatible with the configured source. + """ + root = self._policy.worktree_root + assert root is not None # policy was validated by _expected_identity + nearest = root.resolve() + while not nearest.exists() and nearest != nearest.parent: + nearest = nearest.parent + if not nearest.exists(): + raise ConversationWorktreeError( + "worktree_root has no existing ancestor", phase="policy" + ) + result = self._run_git( + nearest, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + self._policy.create_timeout, + "policy", + ) + if result.returncode == 0: + owner_common_dir = Path(result.stdout.strip()).resolve() + if owner_common_dir != source_common_dir.resolve(): + raise ConversationWorktreeError( + "worktree_root is inside an unrelated repository", phase="policy" + ) + + registered = self._git_stdout( + source, + ["worktree", "list", "--porcelain"], + "policy", + ) + registered_paths = [ + Path(line.removeprefix("worktree ")).resolve() + for line in registered.splitlines() + if line.startswith("worktree ") + ] + primary_path = registered_paths[0] if registered_paths else None + for registered_path in registered_paths: + # Git lists the primary checkout first. A conventional + # /.worktrees root is safe after the common-dir check + # above, while nesting under a linked sibling remains unsafe. + if registered_path == primary_path: + continue + root_is_nested = self._is_within(root, registered_path) + target_is_nested = self._is_within(expected_path, registered_path) + existing_owns_target = ( + existing is not None + and registered_path == expected_path.resolve() + and Path(existing.worktree_path).resolve() == expected_path.resolve() + ) + if root_is_nested or (target_is_nested and not existing_owns_target): + raise ConversationWorktreeError( + "worktree_root or target is inside a registered worktree", + phase="policy", + ) + + def _expected_identity(self, root_session_id: str) -> tuple[Path, str]: + worktree_root = self._policy.worktree_root + if worktree_root is None: + raise ConversationWorktreeError( + "enabled conversation worktree policy has no worktree_root", phase="policy" + ) + digest = hashlib.sha256(root_session_id.encode("utf-8")).hexdigest()[:24] + name = f"conversation-{digest}" + return worktree_root.resolve() / name, f"{self._policy.branch_prefix}/{name}" + + @staticmethod + def _is_within(candidate: Path, parent: Path) -> bool: + try: + candidate.resolve().relative_to(parent.resolve()) + except ValueError: + return False + return True + + def _validate_record_identity( + self, + record: ConversationWorktreeRecord, + *, + path: Path, + branch: str, + repo_common_dir: Path, + ) -> None: + if ( + Path(record.worktree_path).resolve() != path.resolve() + or record.branch != branch + or Path(record.repo_common_dir).resolve() != repo_common_dir.resolve() + ): + raise ConversationWorktreeError( + "conversation worktree identity conflicts with configured source or root", + phase="identity", + ) + + @staticmethod + def _require_recoverable_record(record: ConversationWorktreeRecord) -> None: + if record.state == "removed": + raise ConversationWorktreeError( + "conversation worktree was explicitly removed", phase="recovery" + ) + if record.state not in {"creating", "creation_failed"}: + raise ConversationWorktreeError( + f"conversation worktree is not recoverable from state {record.state!r}", + phase="recovery", + ) + + def _prepare_worktree(self, source: Path, record: ConversationWorktreeRecord) -> None: + """Create or identity-validate the tree while holding the Git lock.""" + self._require_recoverable_record(record) + path = Path(record.worktree_path) + if path.exists(): + try: + self._validate_new_worktree(record) + except ConversationWorktreeError as exc: + if record.state == "creating": + raise ConversationWorktreeError( + "conversation worktree path already exists and is not a matching worktree", + phase="create", + ) from exc + raise + self._ensure_common_owner_claim(record) + self._ensure_owner_marker(record) + self._event("conversation_worktree.reuse", root_session_id=record.root_session_id) + elif record.state == "creation_failed": + raise ConversationWorktreeError( + "failed conversation worktree is missing; retained state requires manual recovery", + phase="recovery", + ) + else: + self._create_worktree(source, record) + + def _ensure_git_worktree_locked( + self, source: Path, record: ConversationWorktreeRecord + ) -> None: + """Keep an active managed tree protected by Git's lifecycle lock.""" + path = Path(record.worktree_path).resolve() + listing = self._git_stdout( + source, + ["worktree", "list", "--porcelain"], + "create", + ) + current_path: Path | None = None + for line in listing.splitlines(): + if line.startswith("worktree "): + current_path = Path(line.removeprefix("worktree ")).resolve() + elif current_path == path and line.startswith("locked"): + return + + # Preserve the lifecycle invariant introduced by #48699 / @JoaoMarcos44: + # a Hermes-owned worktree stays Git-locked until its cleanup owner unlocks it. + self._git_stdout( + source, + [ + "worktree", + "lock", + "--reason", + f"Hermes conversation {record.root_session_id}", + str(path), + ], + "create", + ) + + @contextmanager + def _repository_lock(self, common_dir: Path) -> Iterator[None]: + """Bound one repository's metadata mutation across Hermes processes.""" + with self._lock_path(common_dir / "hermes-conversation-worktree.lock"): + yield + + @contextmanager + def _root_lock(self, common_dir: Path, root_session_id: str) -> Iterator[None]: + """Serialize bootstrap/readiness only for one durable conversation root.""" + digest = hashlib.sha256(root_session_id.encode("utf-8")).hexdigest()[:24] + with self._lock_path(common_dir / f"hermes-conversation-root-{digest}.lock"): + yield + + @contextmanager + def _lock_path(self, lock_path: Path) -> Iterator[None]: + """Take a bounded process + thread lock and normalize setup failures.""" + deadline = time.monotonic() + self._policy.create_timeout + handle = None + locked = False + thread_lock = self._thread_lock(lock_path) + if not thread_lock.acquire(timeout=self._policy.create_timeout): + raise ConversationWorktreeError( + "repository worktree lock timed out", phase="create" + ) + try: + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = self._open_lock_file(lock_path) + handle.seek(0) + handle.write(b"0") + handle.flush() + except OSError as exc: + raise ConversationWorktreeError( + "conversation worktree lock is unavailable", phase="create" + ) from exc + while True: + try: + if os.name == "nt": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + locked = True + break + except OSError: + if time.monotonic() >= deadline: + raise ConversationWorktreeError( + "repository worktree lock timed out", phase="create" + ) + time.sleep(0.05) + yield + finally: + if handle is not None: + if locked: + try: + if os.name == "nt": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: + logger.warning("conversation_worktree.lock_release_failed") + try: + handle.close() + except OSError: + logger.warning("conversation_worktree.lock_close_failed") + thread_lock.release() + + @staticmethod + def _open_lock_file(lock_path: Path): + return lock_path.open("a+b") + + @staticmethod + def _thread_lock(lock_path: Path) -> threading.Lock: + """Return the in-process companion to the cross-process file lock.""" + key = str(lock_path.resolve()) + with _REPOSITORY_THREAD_LOCKS_GUARD: + lock = _REPOSITORY_THREAD_LOCKS.get(key) + if lock is None: + lock = threading.Lock() + _REPOSITORY_THREAD_LOCKS[key] = lock + return lock + + def _create_worktree(self, source: Path, record: ConversationWorktreeRecord) -> None: + path = Path(record.worktree_path) + if path.exists(): + raise ConversationWorktreeError( + "conversation worktree path already exists", phase="create" + ) + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise ConversationWorktreeError( + "conversation worktree parent could not be created", phase="create" + ) from exc + + self._event("conversation_worktree.create", root_session_id=record.root_session_id) + # This common-repository claim is intentionally written *before* Git + # creates the worktree. A crash or failed per-worktree marker write + # after `git worktree add` must still be visible to every generic GC. + self._ensure_common_owner_claim(record) + self._git_stdout( + source, + [ + "worktree", + "add", + "--no-track", + "-b", + record.branch, + str(path), + record.base_commit, + ], + "create", + ) + # Provision the source repository's own runtime before this new + # conversation worktree can be bound to an agent. This is additive: + # an existing destination is never replaced, and a missing source + # environment is left for the configured project bootstrap policy. + bootstrap_worktree_environments(source, path, environment_names=(".venv",)) + self._validate_new_worktree(record) + self._ensure_owner_marker(record) + + def _validate_new_worktree(self, record: ConversationWorktreeRecord) -> None: + path = Path(record.worktree_path) + if not path.is_dir(): + raise ConversationWorktreeError( + "conversation worktree path is missing", phase="validate" + ) + head = self._git_stdout(path, ["rev-parse", "HEAD"], "validate") + branch = self._git_stdout(path, ["branch", "--show-current"], "validate") + common = Path( + self._git_stdout( + path, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + "validate", + ) + ).resolve() + if ( + head != record.base_commit + or branch != record.branch + or common != Path(record.repo_common_dir).resolve() + ): + raise ConversationWorktreeError( + "created conversation worktree failed identity validation", phase="validate" + ) + + def _validated_ready_binding( + self, record: ConversationWorktreeRecord + ) -> ConversationWorktreeBinding: + if record.state != "ready": + raise ConversationWorktreeError( + f"conversation worktree is not ready (state {record.state!r})", + phase="recovery", + ) + _, source_common_dir = self._source_repository_identity() + path, branch = self._expected_identity(record.root_session_id) + self._validate_record_identity( + record, + path=path, + branch=branch, + repo_common_dir=source_common_dir, + ) + if not path.is_dir(): + raise ConversationWorktreeError( + "ready conversation worktree path is missing", phase="recovery" + ) + actual_branch = self._git_stdout(path, ["branch", "--show-current"], "recovery") + actual_common = Path( + self._git_stdout( + path, + ["rev-parse", "--path-format=absolute", "--git-common-dir"], + "recovery", + ) + ).resolve() + if actual_common != source_common_dir: + raise ConversationWorktreeError( + "ready conversation worktree failed identity validation", + phase="recovery", + ) + if actual_branch != record.branch: + # A conversation can legitimately rename its branch while preparing + # or merging a PR. Accept only that narrow drift: the checkout must + # remain on a named branch and both independent owner claims must + # still bind the exact root, path, and common repository. + if not actual_branch or not self._exact_owner_claims_present(record): + raise ConversationWorktreeError( + "ready conversation worktree failed identity validation", + phase="recovery", + ) + ancestor = self._run_git( + path, + ["merge-base", "--is-ancestor", record.base_commit, "HEAD"], + self._policy.create_timeout, + "recovery", + ) + if ancestor.returncode != 0: + # Rebase/squash/reset workflows used while merging PRs can rewrite + # every descendant without changing checkout ownership. Recover + # only when this exact worktree's HEAD reflog proves the recorded + # creation base was previously checked out here, and both durable + # ownership claims still match. A replacement checkout with no + # local continuity proof remains rejected. + if not self._exact_owner_claims_present( + record + ) or not self._worktree_reflog_contains_base(record): + raise ConversationWorktreeError( + "ready conversation worktree no longer descends from its base commit", + phase="recovery", + ) + self._ensure_common_owner_claim(record) + self._ensure_owner_marker(record) + self._event( + "conversation_worktree.reuse", root_session_id=record.root_session_id + ) + binding = self._binding_from_record(record) + if actual_branch != binding.branch: + binding = ConversationWorktreeBinding( + root_session_id=binding.root_session_id, + path=binding.path, + branch=actual_branch, + base_commit=binding.base_commit, + repo_common_dir=binding.repo_common_dir, + ) + return binding + + def _exact_owner_claims_present(self, record: ConversationWorktreeRecord) -> bool: + """Verify both durable ownership claims without repairing either one.""" + path = Path(record.worktree_path) + try: + marker_text = self._git_stdout( + path, ["rev-parse", "--git-path", _OWNER_MARKER], "recovery" + ) + marker = Path(marker_text) + if not marker.is_absolute(): + marker = path / marker + common_claim = _common_owner_claim_path( + Path(record.repo_common_dir).resolve(), path + ) + marker_data = json.loads(marker.read_text(encoding="utf-8")) + common_data = json.loads(common_claim.read_text(encoding="utf-8")) + except (ConversationWorktreeError, OSError, ValueError, TypeError): + return False + return _owner_claim_matches_record( + marker_data, record=record + ) and _owner_claim_matches_record(common_data, record=record) + + def _worktree_reflog_contains_base(self, record: ConversationWorktreeRecord) -> bool: + """Prove rewritten HEAD continuity using this worktree's own reflog.""" + path = Path(record.worktree_path) + try: + reflog_text = self._git_stdout( + path, ["rev-parse", "--git-path", "logs/HEAD"], "recovery" + ) + reflog_path = Path(reflog_text) + if not reflog_path.is_absolute(): + reflog_path = path / reflog_path + lines = reflog_path.read_text(encoding="utf-8").splitlines() + except (ConversationWorktreeError, OSError): + return False + for line in lines: + fields = line.split(None, 2) + if record.base_commit in fields[:2]: + return True + return False + + def _ensure_owner_marker(self, record: ConversationWorktreeRecord) -> None: + path = Path(record.worktree_path) + marker_text = self._git_stdout( + path, ["rev-parse", "--git-path", _OWNER_MARKER], "validate" + ) + marker = Path(marker_text) + if not marker.is_absolute(): + marker = path / marker + _write_owner_claim(marker, self._owner_payload(record)) + + @staticmethod + def _owner_payload(record: ConversationWorktreeRecord) -> dict[str, str]: + return { + "owner": "conversation-worktree-manager", + "root_session_id": record.root_session_id, + "worktree_path": str(Path(record.worktree_path).resolve()), + "repo_common_dir": str(Path(record.repo_common_dir).resolve()), + } + + def _ensure_common_owner_claim(self, record: ConversationWorktreeRecord) -> None: + _write_owner_claim( + _common_owner_claim_path( + Path(record.repo_common_dir).resolve(), Path(record.worktree_path) + ), + self._owner_payload(record), + ) + + @staticmethod + def _remove_common_owner_claim(record: ConversationWorktreeRecord) -> None: + """Drop only the exact durable claim after verified explicit removal.""" + claim = _common_owner_claim_path( + Path(record.repo_common_dir).resolve(), Path(record.worktree_path) + ) + try: + claim.unlink(missing_ok=True) + except OSError: + # Git removal and the durable ledger transition have already + # succeeded. Retaining a stale claim is safe; deleting it later + # is housekeeping, never a reason to misreport cleanup failure. + logger.warning("conversation_worktree.common_claim_remove_failed") + + def _run_bootstrap(self, record: ConversationWorktreeRecord) -> None: + if not self._policy.bootstrap: + return + self._event("conversation_worktree.bootstrap", root_session_id=record.root_session_id) + popen_kwargs = ( + {"creationflags": windows_hide_flags()} + if IS_WINDOWS + else {"process_group": 0} + ) + try: + process = subprocess.Popen( + list(self._policy.bootstrap_command), + cwd=record.worktree_path, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + **popen_kwargs, + ) + except OSError as exc: + raise ConversationWorktreeError( + "bootstrap command could not start", phase="bootstrap" + ) from exc + try: + _stdout, _stderr = process.communicate(timeout=self._policy.bootstrap_timeout) + except subprocess.TimeoutExpired as exc: + # A bootstrap can spawn compilers/package managers and descendants. + # Killing only its direct shell leaves those descendants running in + # the conversation worktree after the failure was recorded. The + # compatibility helper owns POSIX group termination and Windows + # taskkill /T /F cleanup; the bounded drain avoids a hung pipe. + kill_process_tree(process) + try: + process.communicate(timeout=1) + except Exception: + pass + raise ConversationWorktreeError( + f"bootstrap timed out after {self._policy.bootstrap_timeout:g} seconds", + phase="bootstrap", + ) from exc + except Exception as exc: + kill_process_tree(process) + try: + process.communicate(timeout=1) + except Exception: + pass + raise ConversationWorktreeError( + "bootstrap process communication failed", phase="bootstrap" + ) from exc + if process.returncode != 0: + raise ConversationWorktreeError( + f"bootstrap command exited with status {process.returncode}", + phase="bootstrap", + ) + + def _git_stdout(self, cwd: Path, args: list[str], phase: str) -> str: + result = self._run_git(cwd, args, self._timeout_for_phase(phase), phase) + if result.returncode != 0: + raise ConversationWorktreeError( + f"git {args[0]} failed", phase=phase + ) + return result.stdout.strip() + + @staticmethod + def _run_git( + cwd: Path, args: list[str], timeout: float, phase: str + ) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + ["git", "-C", str(cwd), *args], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + check=False, + env=noninteractive_git_env(), + ) + except subprocess.TimeoutExpired as exc: + raise ConversationWorktreeError("git command timed out", phase=phase) from exc + except OSError as exc: + raise ConversationWorktreeError("git command could not start", phase=phase) from exc + + def _timeout_for_phase(self, phase: str) -> float: + return self._policy.create_timeout if phase in {"create", "validate", "identity", "recovery"} else self._policy.create_timeout + + def _record_failure( + self, + root_session_id: str, + record: ConversationWorktreeRecord | None, + error: ConversationWorktreeError, + ) -> None: + if record is None or record.state not in {"creating", "creation_failed"}: + return + try: + self._db.mark_conversation_worktree_failed( + root_session_id, + failure_phase=error.phase, + failure_message=str(error)[:500], + ) + except Exception: + logger.exception("conversation_worktree.failure_record_failed") + + @staticmethod + def _binding_from_record(record: ConversationWorktreeRecord) -> ConversationWorktreeBinding: + return ConversationWorktreeBinding( + root_session_id=record.root_session_id, + path=Path(record.worktree_path), + branch=record.branch, + base_commit=record.base_commit, + repo_common_dir=Path(record.repo_common_dir), + ) + + @staticmethod + def _event(event: str, **fields: str) -> None: + logger.info(event, extra={"conversation_worktree": fields}) diff --git a/agent/conversation_worktree_policy.py b/agent/conversation_worktree_policy.py new file mode 100644 index 0000000000000..a3b117e122515 --- /dev/null +++ b/agent/conversation_worktree_policy.py @@ -0,0 +1,153 @@ +"""Platform-neutral configuration policy for conversation worktree isolation.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import math +from numbers import Real +from pathlib import Path +from typing import Any + + +class ConversationWorktreePolicyError(ValueError): + """Raised when conversation worktree configuration is unsafe or incomplete.""" + + +@dataclass(frozen=True) +class ConversationWorktreePolicy: + enabled: bool + source_worktree: Path | None + worktree_root: Path | None + branch_prefix: str = "hermes/session" + bootstrap: bool = False + bootstrap_command: tuple[str, ...] = () + bootstrap_timeout: float = 300.0 + create_timeout: float = 60.0 + retain_until_explicit_cleanup: bool = True + legacy_location: bool = False + + +_BRANCH_FORBIDDEN = frozenset(" ~^:?*[\\") + + +def _section(config: Mapping[str, object]) -> tuple[Mapping[str, object], bool]: + if "conversation_worktree" in config: + section = config["conversation_worktree"] + if section is None: + # ``DEFAULT_CONFIG`` uses None as a presence sentinel. Deep merge + # preserves it only when no user top-level policy exists, so the + # legacy desktop block remains observable during the migration. + pass + elif not isinstance(section, Mapping): + raise ConversationWorktreePolicyError("conversation_worktree must be a mapping") + else: + return section, False + + desktop = config.get("desktop", {}) + if not isinstance(desktop, Mapping): + raise ConversationWorktreePolicyError("desktop must be a mapping") + section = desktop.get("conversation_worktree", {}) + if not isinstance(section, Mapping): + raise ConversationWorktreePolicyError("desktop.conversation_worktree must be a mapping") + return section, bool(section) + + +def _boolean(section: Mapping[str, object], field: str, default: bool) -> bool: + value = section.get(field, default) + if not isinstance(value, bool): + raise ConversationWorktreePolicyError(f"{field} must be a boolean") + return value + + +def _absolute_path(section: Mapping[str, object], field: str) -> Path | None: + value = section.get(field) + if value is None or value == "": + return None + if not isinstance(value, (str, Path)): + raise ConversationWorktreePolicyError(f"{field} must be an absolute path") + path = Path(value).expanduser() + if not path.is_absolute(): + raise ConversationWorktreePolicyError(f"{field} must be an absolute path") + return path.resolve() + + +def _positive_timeout(section: Mapping[str, object], field: str, default: float) -> float: + value = section.get(field, default) + if isinstance(value, bool) or not isinstance(value, Real): + raise ConversationWorktreePolicyError(f"{field} must be a finite positive number") + value = float(value) + if not math.isfinite(value) or value <= 0: + raise ConversationWorktreePolicyError(f"{field} must be a finite positive number") + return value + + +def _branch_prefix(section: Mapping[str, object]) -> str: + value = section.get("branch_prefix", "hermes/session") + if not isinstance(value, str): + raise ConversationWorktreePolicyError("branch_prefix must be a non-empty safe branch prefix") + prefix = value.strip() + components = prefix.split("/") + if ( + not prefix + or prefix.startswith("/") + or prefix.endswith("/") + or "//" in prefix + or ".." in prefix + or "@{" in prefix + or prefix.endswith(".") + or prefix.endswith(".lock") + or prefix == "@" + or any(component.startswith(".") or component.endswith(".lock") for component in components) + or any(ord(char) < 0x20 or ord(char) == 0x7F for char in prefix) + or any(char in _BRANCH_FORBIDDEN or char.isspace() for char in prefix) + ): + raise ConversationWorktreePolicyError("branch_prefix must be a non-empty safe branch prefix") + return prefix + + +def _bootstrap_command(section: Mapping[str, object]) -> tuple[str, ...]: + value: Any = section.get("bootstrap_command", []) + if not isinstance(value, list) or any(not isinstance(arg, str) or not arg for arg in value): + raise ConversationWorktreePolicyError("bootstrap_command must be a list of non-empty strings") + return tuple(value) + + +def resolve_conversation_worktree_policy(config: Mapping[str, object]) -> ConversationWorktreePolicy: + """Resolve top-level policy, falling back to desktop.conversation_worktree.""" + if not isinstance(config, Mapping): + raise ConversationWorktreePolicyError("configuration must be a mapping") + + section, legacy_location = _section(config) + enabled = _boolean(section, "enabled", False) + source_worktree = _absolute_path(section, "source_worktree") + worktree_root = _absolute_path(section, "worktree_root") + retain_until_explicit_cleanup = _boolean(section, "retain_until_explicit_cleanup", True) + bootstrap = _boolean(section, "bootstrap", False) + bootstrap_command = _bootstrap_command(section) + + if enabled and source_worktree is None: + raise ConversationWorktreePolicyError("source_worktree is required when enabled") + if enabled and worktree_root is None: + raise ConversationWorktreePolicyError("worktree_root is required when enabled") + if enabled and not retain_until_explicit_cleanup: + raise ConversationWorktreePolicyError( + "retain_until_explicit_cleanup must be true when enabled" + ) + if bootstrap and not bootstrap_command: + raise ConversationWorktreePolicyError( + "bootstrap_command must be non-empty when bootstrap is enabled" + ) + + return ConversationWorktreePolicy( + enabled=enabled, + source_worktree=source_worktree, + worktree_root=worktree_root, + branch_prefix=_branch_prefix(section), + bootstrap=bootstrap, + bootstrap_command=bootstrap_command, + bootstrap_timeout=_positive_timeout(section, "bootstrap_timeout", 300.0), + create_timeout=_positive_timeout(section, "create_timeout", 60.0), + retain_until_explicit_cleanup=retain_until_explicit_cleanup, + legacy_location=legacy_location, + ) diff --git a/agent/cross_process_file_lock.py b/agent/cross_process_file_lock.py new file mode 100644 index 0000000000000..f9d05b3c7179e --- /dev/null +++ b/agent/cross_process_file_lock.py @@ -0,0 +1,69 @@ +"""Portable blocking cross-process file lock used by security ledgers.""" + +from __future__ import annotations + +from contextlib import contextmanager +import os +from pathlib import Path +from typing import Iterator + +try: # POSIX + import fcntl as _fcntl +except ImportError: # pragma: no cover - exercised by import simulation + _fcntl = None + +try: # Windows + import msvcrt as _msvcrt +except ImportError: # pragma: no cover - POSIX + _msvcrt = None + + +def secure_file_descriptor_permissions(fd: int) -> None: + """Tighten a descriptor when the platform exposes ``os.fchmod``. + + Windows Python 3.11/3.12 lacks ``os.fchmod``; secure creation flags and + the owning user's ACL remain the platform boundary there. + """ + + fchmod = getattr(os, "fchmod", None) + if fchmod is not None: + fchmod(fd, 0o600) + + +@contextmanager +def exclusive_file_lock(path: Path) -> Iterator[None]: + """Hold one OS-backed lock file across all Hermes processes.""" + + try: + if path.is_symlink(): + raise OSError("cross-process lock path must not be a symlink") + except OSError: + raise + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(path, flags, 0o600) + try: + secure_file_descriptor_permissions(fd) + if _fcntl is not None: + _fcntl.flock(fd, _fcntl.LOCK_EX) + elif _msvcrt is not None: # pragma: no cover - Windows + if os.fstat(fd).st_size == 0: + os.write(fd, b"\0") + os.lseek(fd, 0, os.SEEK_SET) + _msvcrt.locking(fd, _msvcrt.LK_LOCK, 1) + else: + raise OSError("cross-process file locking is unavailable") + yield + finally: + try: + if _fcntl is not None: + _fcntl.flock(fd, _fcntl.LOCK_UN) + elif _msvcrt is not None: # pragma: no cover - Windows + os.lseek(fd, 0, os.SEEK_SET) + _msvcrt.locking(fd, _msvcrt.LK_UNLCK, 1) + finally: + os.close(fd) + + +__all__ = ["exclusive_file_lock", "secure_file_descriptor_permissions"] diff --git a/agent/egress_source_annotations.py b/agent/egress_source_annotations.py new file mode 100644 index 0000000000000..30250746fe502 --- /dev/null +++ b/agent/egress_source_annotations.py @@ -0,0 +1,54 @@ +"""Recognize Python type syntax for the source-bound secret scan only.""" + +from __future__ import annotations + +import ast + + +def _builtin_annotation(node: ast.expr) -> bool: + if isinstance(node, ast.Name): + return node.id in {"str", "bytes", "bool", "int", "float", "object"} + if isinstance(node, ast.Constant): + return node.value is None + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + return _builtin_annotation(node.left) and _builtin_annotation(node.right) + return False + + +def mask_builtin_annotations(text: str) -> str: + """Mask proven annotation syntax, leaving defaults and string contents intact. + + Incomplete snippets and unknown annotations retain the strict original scan. + AST offsets are UTF-8 bytes, including when a line contains non-ASCII names. + """ + + if ":" not in text: + return text + try: + tree = ast.parse(text) + except (SyntaxError, ValueError, RecursionError): + return text + raw = text.encode("utf-8") + offsets = [0] + for line in raw.splitlines(keepends=True): + offsets.append(offsets[-1] + len(line)) + edits = [] + for node in ast.walk(tree): + if isinstance(node, ast.arg) and node.annotation is not None: + start = offsets[node.lineno - 1] + node.col_offset + len(node.arg.encode("utf-8")) + annotation = node.annotation + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + start = offsets[node.target.end_lineno - 1] + node.target.end_col_offset + annotation = node.annotation + else: + continue + if _builtin_annotation(annotation): + end = offsets[annotation.end_lineno - 1] + annotation.end_col_offset + edits.append((start, end)) + pieces = [] + cursor = 0 + for start, end in sorted(edits): + pieces.extend((raw[cursor:start], b" ")) + cursor = end + pieces.append(raw[cursor:]) + return b"".join(pieces).decode("utf-8") diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 505244c541f1a..f5bec1f68ecff 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -40,6 +40,7 @@ class FailoverReason(enum.Enum): model_not_found = "model_not_found" # 404 or invalid model — fallback to different model provider_policy_blocked = "provider_policy_blocked" # Aggregator account data/privacy policy excluded the only endpoint content_policy_blocked = "content_policy_blocked" # Provider safety filter rejected this prompt — don't retry unchanged + egress_policy_blocked = "egress_policy_blocked" # Local privacy firewall denied remote transport — fall back locally without retry format_error = "format_error" # 400 bad request — abort or strip + retry invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type content in tool messages (e.g. Xiaomi MiMo) — downgrade to text and retry @@ -47,6 +48,7 @@ class FailoverReason(enum.Enum): # Provider-specific thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid + unsupported_thinking = "unsupported_thinking" # Selected model has no thinking capability long_context_tier = "long_context_tier" # Anthropic "extra usage" tier gate oauth_long_context_beta_forbidden = "oauth_long_context_beta_forbidden" # Anthropic OAuth rejects 1M beta — disable beta and retry llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern" # llama.cpp grammar rejects regex `pattern`/`format` — strip from tools and retry @@ -278,6 +280,12 @@ def billing_unverified(self) -> bool: "content_filter", "responsibleaipolicyviolation", "new_sensitive", ) +# Local inference server rejects a request that still carries reasoning +# controls when the selected model has no thinking capability. +_UNSUPPORTED_THINKING_PATTERNS = ( + "does not support thinking", "thinking is not supported", "unsupported thinking", +) + # Auth patterns (non-status-code signals). _AUTH_PATTERNS = ( "invalid api key", "invalid_api_key", "gateway_auth_failed", "authentication", "unauthorized", @@ -366,6 +374,10 @@ def _v(reason: FailoverReason, **hints: Any) -> Verdict: _V_AUTH_FALLBACK = _v(_R.auth, **_ABORT_FALLBACK) _V_MODEL_NOT_FOUND = _v(_R.model_not_found, **_ABORT_FALLBACK) _V_CONTENT_BLOCKED = _v(_R.content_policy_blocked, **_ABORT_FALLBACK) +_V_EGRESS_BLOCKED = _v(_R.egress_policy_blocked, retryable=False, should_fallback=False) +# The only two size-based egress reason codes — every other code (secret detection etc.) +# is a security denial and must stay terminal, even mixed with a size code. +_EGRESS_SIZE_REASON_CODES = frozenset({"serialized_bytes_exceeded", "token_cap_exceeded"}) _V_FORMAT_ERROR = _v(_R.format_error, **_ABORT_FALLBACK) _V_POLICY_BLOCKED = _v(_R.provider_policy_blocked, retryable=False) _V_SSL_CERT = _v(_R.ssl_cert_verification, retryable=False) @@ -511,6 +523,13 @@ def _plugin_verdict(c: _Ctx) -> Optional[Verdict]: def _provider_special_cases(c: _Ctx) -> Optional[Verdict]: """Highest-priority provider-specific shapes that a status code would misroute.""" msg, status = c.msg, c.status_code + # A local inference server rejecting a request that still carries reasoning + # controls when the model has no thinking capability is deterministic + # deployment/configuration drift — the local capability probe normally + # prevents this request, so reaching here must never trigger a remote + # fallback (retrying or falling back reproduces the identical rejection). + if any(p in msg for p in _UNSUPPORTED_THINKING_PATTERNS): + return _v(_R.unsupported_thinking, retryable=False, should_fallback=False) # Safety refusal before status classification so a 400 block isn't downgraded # to format_error and a status-less block isn't left retryable (#18028). if any(p in msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS): @@ -558,6 +577,19 @@ def _moa_special_cases(c: _Ctx) -> Optional[Verdict]: return _v(_R.model_not_found, retryable=False) if isinstance(c.error, MoAPresetNotFoundError) else None +def _egress_special_cases(c: _Ctx) -> Optional[Verdict]: + # The local privacy firewall's own denial, not a provider response — distinct + # exception type, checked ahead of every status/message-based stage. + from agent.llm_egress_firewall import EgressBlocked + + if not isinstance(c.error, EgressBlocked): + return None + reason_codes = c.error.decision.reason_codes + if reason_codes and set(reason_codes) <= _EGRESS_SIZE_REASON_CODES: + return {**_V_PAYLOAD_TOO_LARGE, "error_context": {"reason_codes": reason_codes}} + return {**_V_EGRESS_BLOCKED, "error_context": {"reason_codes": reason_codes}} + + def _by_error_code(c: _Ctx) -> Optional[Verdict]: """Structured error codes from the response body.""" # Request-validation failure as plain-text ``event: error`` SSE data behind @@ -624,7 +656,7 @@ def _by_status(c: _Ctx) -> Optional[Verdict]: # MoA shapes → structured error code → message patterns → SSL → disconnect + # large session → transport types → unknown (retryable with backoff). _STAGES: Sequence[Callable[[_Ctx], Optional[Verdict]]] = ( - _plugin_verdict, _provider_special_cases, _by_status, _moa_special_cases, + _egress_special_cases, _plugin_verdict, _provider_special_cases, _by_status, _moa_special_cases, _by_error_code, _by_message, _by_transport, ) diff --git a/agent/kanban_stop.py b/agent/kanban_stop.py index 5f4669bdb8506..f957ada131a7a 100644 --- a/agent/kanban_stop.py +++ b/agent/kanban_stop.py @@ -6,13 +6,67 @@ from __future__ import annotations +import json import os from typing import Any, Iterable, Optional -_TERMINAL_KANBAN_TOOLS = frozenset({"kanban_complete", "kanban_block"}) +_TERMINAL_KANBAN_TOOLS = frozenset( + { + "kanban_complete", + "kanban_block", + "kanban_request_review", + "kanban_request_changes", + } +) _DEFAULT_MAX_ATTEMPTS = 2 +_MAX_REVIEW_SUMMARY_CHARS = 4000 + + +def _auto_review_on_stop_enabled() -> bool: + """Return whether unverified worker prose may enter the review lane. + + A missing terminal Kanban action is normally a protocol violation, not + evidence that a reviewer can use. Retrying the original assignee keeps + that worker accountable for its receipt and prevents unrelated review + workers from becoming a queue sink. Deployments that deliberately want + the old evidence-preservation behavior can opt in per worker or config. + """ + + raw = os.environ.get("HERMES_KANBAN_AUTO_REVIEW_ON_STOP") + if raw is not None: + return raw.strip().lower() in {"1", "true", "yes", "on"} + try: + from hermes_cli.config import load_config + + kanban = load_config().get("kanban") or {} + return kanban.get("auto_review_on_stop") is True + except Exception: + return False + + +def _configured_review_profile() -> str | None: + """Return an installed independent reviewer configured for stop handoffs.""" + + configured = (os.environ.get("HERMES_KANBAN_REVIEWER_PROFILE") or "").strip() + if not configured: + try: + from hermes_cli.config import load_config + + kanban = load_config().get("kanban") or {} + configured = str(kanban.get("reviewer_profile") or "").strip() + except Exception: + return None + if not configured: + return None + try: + from hermes_constants import get_default_hermes_root + + profile_dir = get_default_hermes_root() / "profiles" / configured + return configured if profile_dir.is_dir() else None + except Exception: + return None def kanban_stop_nudge_enabled() -> bool: @@ -22,6 +76,48 @@ def kanban_stop_nudge_enabled() -> bool: return bool((os.environ.get("HERMES_KANBAN_TASK") or "").strip()) +def kanban_shutdown_drain_requested() -> bool: + """Return whether this worker was asked to pause at a turn boundary.""" + if not (os.environ.get("HERMES_KANBAN_TASK") or "").strip(): + return False + try: + from hermes_cli import kanban_db + + return kanban_db.shutdown_drain_requested() + except Exception: + # A missing/unreadable control path must not turn a normal worker + # response into a shutdown protocol failure. The dispatcher reclaim + # remains the bounded fallback when the cooperative path is unavailable. + return False + + +def pause_current_kanban_run(*, reason: str = "dispatcher shutdown") -> bool: + """Release this worker's active run so the card can be resumed later.""" + task_id = (os.environ.get("HERMES_KANBAN_TASK") or "").strip() + raw_run_id = (os.environ.get("HERMES_KANBAN_RUN_ID") or "").strip() + claim_lock = (os.environ.get("HERMES_KANBAN_CLAIM_LOCK") or "").strip() + if not task_id or not raw_run_id or not claim_lock: + return False + try: + run_id = int(raw_run_id) + from hermes_cli.kanban_db_connect import connect + from hermes_cli.kanban_db_recovery import pause_task + + conn = connect() + try: + return pause_task( + conn, + task_id, + expected_run_id=run_id, + claimer=claim_lock, + reason=reason, + ) + finally: + conn.close() + except Exception: + return False + + def _tool_call_name(tc: Any) -> str: """Tool name from a dict or object tool call (``function.name`` first, then ``name``).""" if isinstance(tc, dict): @@ -44,6 +140,63 @@ def session_called_kanban_terminal(messages: Iterable[dict] | None) -> bool: return False +def successful_kanban_terminal_transition( + *, + messages: Iterable[dict] | None, + tool_calls: Iterable[Any] | None, +) -> bool: + """Return whether this worker's current tool batch durably transitioned it. + + The conversation loop calls this only after the executor has persisted + every tool-result row. Match the current batch by tool-call id and require + the canonical Kanban ``{"ok": true}`` response; merely attempting a + terminal tool (or receiving an error) must not stop the worker before it + can correct the handoff. + """ + if not kanban_stop_nudge_enabled(): + return False + try: + from agent.delegation_context import is_dispatcher_owned_worker_context + + if not is_dispatcher_owned_worker_context(): + return False + except Exception: + return False + + terminal_ids: set[str] = set() + for tool_call in tool_calls or []: + if _tool_call_name(tool_call) not in _TERMINAL_KANBAN_TOOLS: + continue + if isinstance(tool_call, dict): + call_id = tool_call.get("id") or tool_call.get("tool_call_id") + else: + call_id = getattr(tool_call, "id", None) + if call_id: + terminal_ids.add(str(call_id)) + if not terminal_ids: + return False + + for message in messages or []: + if not isinstance(message, dict) or message.get("role") != "tool": + continue + if str(message.get("tool_call_id") or "") not in terminal_ids: + continue + if str(message.get("name") or message.get("tool_name") or "") not in ( + _TERMINAL_KANBAN_TOOLS + ): + continue + content = message.get("content") + if not isinstance(content, str): + continue + try: + payload = json.loads(content) + except (json.JSONDecodeError, TypeError): + continue + if isinstance(payload, dict) and payload.get("ok") is True: + return True + return False + + def build_kanban_stop_nudge( *, messages: Iterable[dict] | None = None, @@ -76,4 +229,31 @@ def build_kanban_stop_nudge( ) +def reconcile_kanban_stop_to_review( + *, messages: Iterable[dict] | None, final_response: Any, + attempts: int, max_attempts: int = _DEFAULT_MAX_ATTEMPTS, +) -> bool: + """Preserve bounded final evidence through the configured independent review gate.""" + if (not kanban_stop_nudge_enabled() or not _auto_review_on_stop_enabled() + or attempts < max_attempts or session_called_kanban_terminal(messages)): + return False + response = str(final_response or "").strip() + reviewer = _configured_review_profile() + if not response or reviewer is None: + return False + try: + from tools.kanban_tools import _handle_request_review + raw = _handle_request_review({ + "summary": f"Automatic terminal handoff after {attempts} unanswered Kanban stop " + f"nudges. Worker final output:\n\n{response[:4000]}", + "reviewer": reviewer, + "metadata": {"source": "kanban_stop_guard", "terminal_nudges": attempts, + "completion_inferred": False}, + }) + payload = json.loads(raw) if isinstance(raw, str) else raw + return isinstance(payload, dict) and payload.get("ok") is True + except Exception: + return False + + __all__ = ["build_kanban_stop_nudge", "kanban_stop_nudge_enabled", "session_called_kanban_terminal"] diff --git a/agent/learning_graph.py b/agent/learning_graph.py index c653107301e3b..1db34521054b2 100644 --- a/agent/learning_graph.py +++ b/agent/learning_graph.py @@ -1,16 +1,22 @@ """Assemble the "learning made visible" graph for desktop. -Scoped to what a user actually learns over time: non-base, learned/profile -skills (agent-created or used) plus ``MEMORY.md`` / ``USER.md`` chunks as -first-class nodes. Skill links come from declared ``related_skills``; -memory→skill links are derived from lexical overlap. +This graph is intentionally scoped to what a user actually learns over time: +- non-base, learned/profile skills (agent-created or used), +- memory chunks from ``MEMORY.md`` / ``USER.md`` as first-class nodes. + +Skill links come from declared ``related_skills``. Memory-to-skill links are +derived from lexical overlap so the graph can answer "which learned skills are +connected to the things I remember?". + +Run as a module to print edge-density stats against real data: + + python -m agent.learning_graph """ from __future__ import annotations import json import re -from collections import Counter from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path @@ -18,9 +24,6 @@ from hermes_constants import get_hermes_home -_SKIP_PARTS = {".archive", ".hub", "node_modules", ".git"} -_USAGE_TS_KEYS = ("last_activity_at", "last_used_at", "last_viewed_at", "last_patched_at", "created_at") - @dataclass class SkillNode: @@ -35,91 +38,152 @@ class SkillNode: related: list[str] = field(default_factory=list) -def _fm_field(fm: dict[str, Any], key: str) -> Any: - """Top-level ``key`` or ``metadata.hermes.``; tolerant of the string-valued - frontmatter that ``parse_frontmatter``'s malformed-YAML fallback produces.""" - if fm.get(key): - return fm[key] +def _frontmatter(text: str) -> dict[str, Any]: + try: + from agent.skill_utils import parse_frontmatter + + fm, _ = parse_frontmatter(text) + return fm or {} + except Exception: + return {} + + +def _hermes_meta(fm: dict[str, Any]) -> dict[str, Any]: + """``metadata.hermes`` as a dict, tolerant of the string-valued frontmatter + that ``parse_frontmatter``'s malformed-YAML fallback produces.""" meta = fm.get("metadata") hermes = meta.get("hermes") if isinstance(meta, dict) else None - return hermes.get(key) if isinstance(hermes, dict) else None + return hermes if isinstance(hermes, dict) else {} def _related(fm: dict[str, Any]) -> list[str]: - raw = _fm_field(fm, "related_skills") - raw = raw.strip("[]").split(",") if isinstance(raw, str) else raw - return [str(r).strip() for r in raw if str(r).strip()] if isinstance(raw, list) else [] + raw = fm.get("related_skills") or _hermes_meta(fm).get("related_skills") + if isinstance(raw, list): + return [str(r).strip() for r in raw if str(r).strip()] + if isinstance(raw, str): + return [r.strip() for r in raw.strip("[]").split(",") if r.strip()] + return [] + + +def _category(fm: dict[str, Any], skill_md: Path) -> str: + cat = fm.get("category") or _hermes_meta(fm).get("category") + if cat: + return str(cat) + # …/skills///SKILL.md + parts = skill_md.parts + return parts[-3] if len(parts) >= 3 else "general" + + +def _iter_skill_files(roots: list[tuple[str, Path]]): + for source, root in roots: + if root.exists(): + for path in root.rglob("SKILL.md"): + yield source, path def _load_usage() -> dict[str, dict[str, Any]]: try: from tools.skill_usage import load_usage + return load_usage() except Exception: + path = get_hermes_home() / "skills" / ".usage.json" try: - return json.loads((get_hermes_home() / "skills" / ".usage.json").read_text(encoding="utf-8")) + return json.loads(path.read_text(encoding="utf-8")) except Exception: return {} def _to_int_ts(value: Any) -> Optional[int]: - """Epoch seconds from a number, numeric string, or ISO timestamp; None otherwise.""" try: - if value is None or not (s := str(value).strip()): + if value is None: return None if isinstance(value, (int, float)): return int(value) + s = str(value).strip() + if not s: + return None try: return int(float(s)) except ValueError: parsed = datetime.fromisoformat(s.replace("Z", "+00:00")) - return int((parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)).timestamp()) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return int(parsed.timestamp()) except Exception: return None +def _usage_timestamp(rec: dict[str, Any]) -> Optional[int]: + for key in ("last_activity_at", "last_used_at", "last_viewed_at", "last_patched_at", "created_at"): + ts = _to_int_ts(rec.get(key)) + if ts is not None: + return ts + return None + + def build_skill_nodes(skill_roots: list[tuple[str, Path]]) -> dict[str, SkillNode]: usage = _load_usage() nodes: dict[str, SkillNode] = {} - for source, root in skill_roots: - for skill_md in root.rglob("SKILL.md") if root.exists() else (): - if _SKIP_PARTS.intersection(skill_md.parts): - continue - try: - text = skill_md.read_text(encoding="utf-8")[:4000] - except OSError: - continue - try: - from agent.skill_utils import parse_frontmatter - fm = parse_frontmatter(text)[0] or {} - except Exception: - fm = {} - name = str(fm.get("name") or skill_md.parent.name).strip() - if not name or name in nodes: - continue - rec, cat, parts = usage.get(name, {}), _fm_field(fm, "category"), skill_md.parts # …/skills///SKILL.md - usage_ts = next((ts for ts in (_to_int_ts(rec.get(k)) for k in _USAGE_TS_KEYS) if ts is not None), None) - nodes[name] = SkillNode( - name=name, category=str(cat) if cat else parts[-3] if len(parts) >= 3 else "general", source=source, - timestamp=usage_ts or _to_int_ts(skill_md.stat().st_mtime), - use_count=int(rec.get("use_count", 0) or 0), state=str(rec.get("state", "active") or "active"), - created_by=rec.get("created_by"), pinned=bool(rec.get("pinned", False)), related=_related(fm), - ) + + for source, skill_md in _iter_skill_files(skill_roots): + if any(p in {".archive", ".hub", "node_modules", ".git"} for p in skill_md.parts): + continue + try: + fm = _frontmatter(skill_md.read_text(encoding="utf-8")[:4000]) + except OSError: + continue + name = str(fm.get("name") or skill_md.parent.name).strip() + if not name or name in nodes: + continue + rec = usage.get(name, {}) + last_activity = _usage_timestamp(rec) + file_ts = _to_int_ts(skill_md.stat().st_mtime) + nodes[name] = SkillNode( + name=name, + category=_category(fm, skill_md), + source=source, + timestamp=last_activity or file_ts, + use_count=int(rec.get("use_count", 0) or 0), + state=str(rec.get("state", "active") or "active"), + created_by=rec.get("created_by"), + pinned=bool(rec.get("pinned", False)), + related=_related(fm), + ) return nodes def build_edges(nodes: dict[str, SkillNode]) -> list[tuple[str, str]]: - """Undirected related_skills edges where BOTH endpoints exist (deduped, first-seen order).""" - return list(dict.fromkeys( - (min(node.name, target), max(node.name, target)) for node in nodes.values() for target in node.related if target in nodes and target != node.name - )) + """Undirected related_skills edges where BOTH endpoints exist (deduped).""" + seen: set[tuple[str, str]] = set() + edges: list[tuple[str, str]] = [] + for node in nodes.values(): + for target in node.related: + if target in nodes and target != node.name: + a, b = sorted((node.name, target)) + key = (a, b) + if key not in seen: + seen.add(key) + edges.append(key) + return edges def density_stats(nodes: dict[str, SkillNode], edges: list[tuple[str, str]]) -> dict[str, Any]: - linked, cats, n = {x for edge in edges for x in edge}, Counter(x.category for x in nodes.values()), len(nodes) or 1 + linked: set[str] = set() + for a, b in edges: + linked.add(a) + linked.add(b) + cats: dict[str, int] = {} + for n in nodes.values(): + cats[n.category] = cats.get(n.category, 0) + 1 + n = len(nodes) or 1 return { - "nodes": len(nodes), "related_edges": len(edges), "edges_per_node": round(len(edges) / n, 3), - "linked_nodes": len(linked), "isolated_pct": round(100 * (n - len(linked)) / n, 1), "categories": len(cats), + "nodes": len(nodes), + "related_edges": len(edges), + "edges_per_node": round(len(edges) / n, 3), + "linked_nodes": len(linked), + "isolated_pct": round(100 * (n - len(linked)) / n, 1), + "categories": len(cats), "agent_created": sum(1 for x in nodes.values() if x.created_by == "agent"), "used": sum(1 for x in nodes.values() if x.use_count > 0), "top_categories": sorted(cats.items(), key=lambda kv: -kv[1])[:8], @@ -127,23 +191,32 @@ def density_stats(nodes: dict[str, SkillNode], edges: list[tuple[str, str]]) -> def _memory_cards() -> list[dict[str, Any]]: - """``MEMORY.md`` / ``USER.md`` prose split on bare ``§`` separators; every - non-empty chunk becomes one card (MEMORY.md cards first, then USER.md).""" + """Freeform memory as readable cards. + + ``MEMORY.md`` / ``USER.md`` are prose split on bare ``§`` separators; each + chunk becomes one card. Every chunk is surfaced — the graph shows everything. + """ base = get_hermes_home() / "memories" cards: list[dict[str, Any]] = [] for fname, source in (("MEMORY.md", "memory"), ("USER.md", "profile")): path = base / fname try: - text, file_ts = path.read_text(encoding="utf-8").strip(), _to_int_ts(path.stat().st_mtime) + text = path.read_text(encoding="utf-8").strip() + file_ts = _to_int_ts(path.stat().st_mtime) except OSError: continue for chunk_idx, chunk in enumerate(c.strip() for c in text.split("\n§\n")): - if chunk: - first = chunk.splitlines()[0].strip().lstrip("# ").strip() - cards.append({ - "source": source, "timestamp": file_ts + chunk_idx if file_ts is not None else None, - "title": (first[:80] + "…") if len(first) > 80 else first, "body": chunk[:1200], - }) + if not chunk: + continue + first = chunk.splitlines()[0].strip().lstrip("# ").strip() + cards.append( + { + "source": source, + "timestamp": file_ts + chunk_idx if file_ts is not None else None, + "title": (first[:80] + "…") if len(first) > 80 else first, + "body": chunk[:1200], + } + ) return cards @@ -152,55 +225,225 @@ def _tokenize(text: str) -> set[str]: def _memory_skill_edges(memory_cards: list[dict[str, Any]], skills: list[SkillNode]) -> list[tuple[str, str]]: - """Top-4 lexically overlapping skills per memory card (name hit weighs 6).""" edges: list[tuple[str, str]] = [] - skill_meta = [(s.name, _tokenize(s.name), s.name.lower()) for s in skills] + skill_meta = [(s, _tokenize(s.name), s.name.lower()) for s in skills] for idx, card in enumerate(memory_cards): + mem_id = f"memory:{card['source']}:{idx}" text = f"{card.get('title', '')}\n{card.get('body', '')}".lower() text_tokens = _tokenize(text) - scored = sorted( - ((score, name) for name, tokens, name_lower in skill_meta if (score := (6 if name_lower in text else 0) + len(tokens & text_tokens)) > 0), - key=lambda x: (-x[0], x[1]), - ) - edges.extend((f"memory:{card['source']}:{idx}", name) for _, name in scored[:4]) + scored: list[tuple[int, str]] = [] + for skill, tokens, skill_name_lower in skill_meta: + score = 0 + if skill_name_lower in text: + score += 6 + score += len(tokens & text_tokens) + if score > 0: + scored.append((score, skill.name)) + scored.sort(key=lambda x: (-x[0], x[1])) + for _, skill_name in scored[:4]: + edges.append((mem_id, skill_name)) return edges +def _skill_roots() -> list[tuple[str, Path]]: + repo = Path(__file__).resolve().parent.parent + home_skills = get_hermes_home() / "skills" + return [("base", repo / "skills"), ("profile", home_skills)] + + +def _shared_catalog_config() -> tuple[bool, Path | None, list[str]]: + try: + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() or {} + learning = config.get("learning") or {} + if not isinstance(learning, dict) or not bool( + learning.get("shared_catalog_enabled") + ): + return False, None, [] + raw_vault = str(learning.get("vault_dir") or "").strip() + if not raw_vault: + return True, None, ["vault_dir_required"] + vault = Path(raw_vault).expanduser() + if not vault.is_absolute(): + return True, None, ["vault_dir_not_absolute"] + return True, vault, [] + except Exception: + return True, None, ["shared_catalog_config_unreadable"] + + +def _shared_catalog(): + enabled, vault, diagnostics = _shared_catalog_config() + if not enabled: + return False, [], diagnostics + if vault is None: + return True, [], diagnostics + try: + from agent.learning_vault import read_vault_learning + + nodes, read_diagnostics = read_vault_learning(vault) + diagnostics.extend(item.reason_code for item in read_diagnostics[:100]) + return True, nodes, diagnostics + except Exception: + diagnostics.append("shared_catalog_unreadable") + return True, [], diagnostics + + +def _vault_node_id(node) -> str: + prefix = "vault-skill" if node.kind == "skill-reference" else "vault-memory" + return f"{prefix}:{node.record_id}" + + +def _shared_edges( + shared_nodes, learned_skills: dict[str, SkillNode] +) -> list[tuple[str, str]]: + ids = {node.record_id: _vault_node_id(node) for node in shared_nodes} + edges: set[tuple[str, str]] = set() + for node in shared_nodes: + source = ids[node.record_id] + for related_id in node.related_record_ids: + target = ids.get(related_id) + if target and target != source: + edges.add((source, target)) + + skill_meta = [ + (skill.name, _tokenize(skill.name), skill.name.lower()) + for skill in learned_skills.values() + ] + for node in shared_nodes: + if node.kind != "shared-memory": + continue + text = f"{node.label}\n{node.summary}".lower() + tokens = _tokenize(text) + scored: list[tuple[int, str]] = [] + for name, skill_tokens, lower_name in skill_meta: + score = (6 if lower_name in text else 0) + len(tokens & skill_tokens) + if score > 0: + scored.append((score, name)) + for _, name in sorted(scored, key=lambda item: (-item[0], item[1]))[:2]: + edges.add((_vault_node_id(node), name)) + return sorted(edges) + + def build_learning_graph() -> dict[str, Any]: - """Full payload for the desktop learning panel: non-base skills with real - learning signal (agent-created or used) plus memory chunks as graph nodes.""" - roots = [("base", Path(__file__).resolve().parent.parent / "skills"), ("profile", get_hermes_home() / "skills")] + """Full payload for the desktop learning panel. + + Focus on what is profile-learned and actionable: + - skills that are NOT base-installed and show real learning signal + (agent-created or used), + - memory chunks as first-class graph nodes connected to those learned skills. + """ + all_skills = build_skill_nodes(_skill_roots()) learned_skills = { - name: node for name, node in build_skill_nodes(roots).items() + name: node + for name, node in all_skills.items() if node.source != "base" and (node.created_by == "agent" or node.use_count > 0) } - skill_edges, memory_cards = build_edges(learned_skills), _memory_cards() + skill_edges = build_edges(learned_skills) + memory_cards = _memory_cards() memory_edges = _memory_skill_edges(memory_cards, list(learned_skills.values())) - clusters = Counter(node.category for node in learned_skills.values()) + shared_enabled, shared_nodes, shared_diagnostics = _shared_catalog() + shared_edges = _shared_edges(shared_nodes, learned_skills) + + edges = skill_edges + memory_edges + shared_edges + clusters: dict[str, int] = {} + for node in learned_skills.values(): + clusters[node.category] = clusters.get(node.category, 0) + 1 if memory_cards: clusters["memory"] = len(memory_cards) + for node in shared_nodes: + clusters[f"shared:{node.area}"] = clusters.get(f"shared:{node.area}", 0) + 1 + origin = f"shared-origin:{node.origin_agent}" + clusters[origin] = clusters.get(origin, 0) + 1 + status = f"shared-status:{node.status}" + clusters[status] = clusters.get(status, 0) + 1 graph_nodes = [ { - "id": n.name, "label": n.name, "kind": "skill", "timestamp": n.timestamp, "category": n.category, - "useCount": n.use_count, "state": n.state, "createdBy": n.created_by, "pinned": n.pinned, + "id": n.name, + "label": n.name, + "kind": "skill", + "timestamp": n.timestamp, + "category": n.category, + "useCount": n.use_count, + "state": n.state, + "createdBy": n.created_by, + "pinned": n.pinned, } for n in learned_skills.values() - ] + [ - { - "id": f"memory:{card['source']}:{i}", "label": card["title"], "kind": "memory", - "memorySource": card["source"], "timestamp": card.get("timestamp"), "category": "memory", - "useCount": 0, "state": "active", "createdBy": "memory", "pinned": False, - } - for i, card in enumerate(memory_cards) ] - return { + for i, card in enumerate(memory_cards): + graph_nodes.append( + { + "id": f"memory:{card['source']}:{i}", + "label": card["title"], + "kind": "memory", + "memorySource": card["source"], + "timestamp": card.get("timestamp"), + "category": "memory", + "useCount": 0, + "state": "active", + "createdBy": "memory", + "pinned": False, + } + ) + for node in shared_nodes: + graph_nodes.append({ + "id": _vault_node_id(node), + "label": node.label, + "kind": node.kind, + "originAgent": node.origin_agent, + "area": node.area, + "status": node.status, + "executionStatus": node.execution_status, + "timestamp": node.timestamp, + "category": f"shared:{node.area}", + "useCount": 0, + "state": "active", + "createdBy": node.origin_agent, + "pinned": False, + }) + + payload = { "nodes": graph_nodes, - "edges": [{"source": a, "target": b} for a, b in skill_edges + memory_edges], - "clusters": [{"category": c, "count": n} for c, n in sorted(clusters.items(), key=lambda kv: -kv[1])], + "edges": [{"source": a, "target": b} for a, b in edges], + "clusters": [ + {"category": c, "count": n} + for c, n in sorted(clusters.items(), key=lambda kv: -kv[1]) + ], "memory": memory_cards, "stats": { **density_stats(learned_skills, skill_edges), - "memory_nodes": len(memory_cards), "memory_skill_edges": len(memory_edges), "learned_skills": len(learned_skills), + "memory_nodes": len(memory_cards), + "memory_skill_edges": len(memory_edges), + "learned_skills": len(learned_skills), }, } + if shared_enabled: + by_origin: dict[str, int] = {} + by_area: dict[str, int] = {} + by_status: dict[str, int] = {} + by_kind: dict[str, int] = {} + for node in shared_nodes: + for counts, key in ( + (by_origin, node.origin_agent), + (by_area, node.area), + (by_status, node.status), + (by_kind, node.kind), + ): + counts[key] = counts.get(key, 0) + 1 + payload["shared_catalog"] = { + "nodes": len(shared_nodes), + "edges": len(shared_edges), + "byOrigin": by_origin, + "byArea": by_area, + "byStatus": by_status, + "byKind": by_kind, + "diagnostics": shared_diagnostics, + } + return payload + + +if __name__ == "__main__": + nodes = build_skill_nodes(_skill_roots()) + print(json.dumps(density_stats(nodes, build_edges(nodes)), indent=2)) diff --git a/agent/learning_vault.py b/agent/learning_vault.py new file mode 100644 index 0000000000000..557adc51f07bf --- /dev/null +++ b/agent/learning_vault.py @@ -0,0 +1,205 @@ +"""Bounded, read-only adapter for the shared agent-learning Vault catalog.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Optional + +from agent.skill_utils import parse_frontmatter + + +MAX_NOTE_BYTES = 16 * 1024 +MAX_SUMMARY_CHARS = 1200 +MAX_CATALOG_FILES = 5000 +_AUTHORITIES = {"source-index", "narrative-only"} +_AGENTS = {"claude", "codex", "hermes", "shared"} +_KINDS = {"memory", "working-preference", "skill-reference"} + + +@dataclass(frozen=True) +class VaultLearningNode: + record_id: str + label: str + kind: str + origin_agent: str + area: str + status: str + timestamp: Optional[int] + related_record_ids: tuple[str, ...] + summary: str + execution_status: str + + +@dataclass(frozen=True) +class VaultLearningDiagnostic: + reason_code: str + source_label: str = "" + + +def _text(value: object) -> str: + return str(value or "").strip() + + +def _truthy(value: object) -> bool: + if isinstance(value, bool): + return value + return _text(value).lower() in {"1", "true", "yes", "on"} + + +def _strings(value: object) -> tuple[str, ...]: + if isinstance(value, list): + return tuple(_text(item) for item in value if _text(item)) + if isinstance(value, str): + stripped = value.strip() + if stripped.startswith("[") and stripped.endswith("]"): + return tuple( + _text(item) for item in stripped[1:-1].split(",") if _text(item) + ) + return () + + +def _timestamp(metadata: dict[str, Any], path: Path) -> Optional[int]: + for key in ("verified_at", "observed_at", "created_at"): + value = _text(metadata.get(key)) + if not value: + continue + try: + from datetime import datetime, timezone + + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return int(parsed.timestamp()) + except (ValueError, OverflowError): + continue + try: + return int(path.stat().st_mtime) + except OSError: + return None + + +def _catalog_roots(vault: Path) -> Iterable[Path]: + for agent in ("Claude", "Codex", "Hermes", "Shared"): + yield vault / "Memories" / agent + for agent in ("Claude", "Codex", "Hermes"): + yield vault / "Reference" / "Agent Skills" / agent + + +def _catalog_files(vault: Path) -> Iterable[Path]: + seen = 0 + for root in _catalog_roots(vault): + if not root.exists(): + continue + for path in sorted(root.rglob("*.md")): + if seen >= MAX_CATALOG_FILES: + return + seen += 1 + yield path + + +def _relative_label(path: Path, vault: Path) -> str: + try: + return path.resolve().relative_to(vault).as_posix() + except (OSError, ValueError): + return path.name + + +def _read_note( + path: Path, vault: Path +) -> tuple[str | None, VaultLearningDiagnostic | None]: + label = _relative_label(path, vault) + try: + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(vault) or not resolved.is_file(): + return None, VaultLearningDiagnostic("note_outside_vault", label) + if resolved.stat().st_size > MAX_NOTE_BYTES: + return None, VaultLearningDiagnostic("note_too_large", label) + return resolved.read_text(encoding="utf-8"), None + except UnicodeDecodeError: + return None, VaultLearningDiagnostic("invalid_utf8", label) + except OSError: + return None, VaultLearningDiagnostic("note_unreadable", label) + + +def _node_from_note( + path: Path, vault: Path +) -> tuple[VaultLearningNode | None, VaultLearningDiagnostic | None]: + raw, diagnostic = _read_note(path, vault) + if raw is None: + return None, diagnostic + metadata, body = parse_frontmatter(raw) + label = _relative_label(path, vault) + record_id = _text(metadata.get("record_id")) + if not record_id: + return None, VaultLearningDiagnostic("record_id_required", label) + if _text(metadata.get("schema_name")) != "agent_learning_record_v1": + return None, VaultLearningDiagnostic("unsupported_schema", label) + if not _truthy(metadata.get("sync_owned")): + return None, VaultLearningDiagnostic("sync_owned_required", label) + if _text(metadata.get("classification")) != "diagnostic-only": + return None, VaultLearningDiagnostic("unsupported_classification", label) + if _text(metadata.get("authority")) not in _AUTHORITIES: + return None, VaultLearningDiagnostic("unsupported_authority", label) + source_kind = _text(metadata.get("kind")) + if source_kind not in _KINDS: + return None, VaultLearningDiagnostic("unsupported_kind", label) + origin_agent = _text(metadata.get("agent")).lower() + if origin_agent not in _AGENTS: + return None, VaultLearningDiagnostic("unsupported_agent", label) + execution_status = _text(metadata.get("execution_status")) + if source_kind == "skill-reference" and execution_status != "reference-only": + return None, VaultLearningDiagnostic("skill_not_reference_only", label) + summary = body.strip()[:MAX_SUMMARY_CHARS] + graph_kind = ( + "skill-reference" if source_kind == "skill-reference" else "shared-memory" + ) + return ( + VaultLearningNode( + record_id=record_id, + label=_text(metadata.get("title")) or path.stem, + kind=graph_kind, + origin_agent=origin_agent, + area=_text(metadata.get("memory_area")) or "General", + status=_text(metadata.get("status")) or "unknown", + timestamp=_timestamp(metadata, path), + related_record_ids=_strings(metadata.get("related_record_ids")), + summary=summary, + execution_status=execution_status or "not-applicable", + ), + None, + ) + + +def read_vault_learning( + vault_dir: Path, +) -> tuple[list[VaultLearningNode], list[VaultLearningDiagnostic]]: + """Read allowlisted normalized notes; individual failures stay diagnostic.""" + + try: + vault = vault_dir.expanduser().resolve(strict=True) + except (OSError, RuntimeError): + return [], [VaultLearningDiagnostic("vault_unavailable")] + if not vault.is_dir(): + return [], [VaultLearningDiagnostic("vault_unavailable")] + + nodes: list[VaultLearningNode] = [] + diagnostics: list[VaultLearningDiagnostic] = [] + seen: set[str] = set() + for path in _catalog_files(vault): + node, diagnostic = _node_from_note(path, vault) + if diagnostic is not None: + diagnostics.append(diagnostic) + continue + if node is None: + continue + if node.record_id in seen: + diagnostics.append( + VaultLearningDiagnostic( + "duplicate_record_id", _relative_label(path, vault) + ) + ) + continue + seen.add(node.record_id) + nodes.append(node) + return nodes, diagnostics diff --git a/agent/llm_egress_firewall.py b/agent/llm_egress_firewall.py new file mode 100644 index 0000000000000..322c4320c7d13 --- /dev/null +++ b/agent/llm_egress_firewall.py @@ -0,0 +1,2519 @@ +"""Source-bound policy checks for outbound LLM requests. + +The firewall is deliberately transport-agnostic. Callers hand it the final +logical request and resolved route immediately before invoking a provider. +It returns an immutable allow decision, or raises :class:`EgressBlocked` with +an immutable block decision. +""" + +from __future__ import annotations + +import base64 +import binascii +import ipaddress +import json +import math +import os +import re +import stat +from dataclasses import asdict, dataclass, replace +from enum import StrEnum +from hmac import compare_digest +from hashlib import sha256 +from pathlib import Path +from typing import Any, Mapping, Sequence +from urllib.parse import urlsplit + +from agent.file_safety import get_read_block_error +from agent.egress_source_annotations import mask_builtin_annotations +from agent.cross_process_file_lock import ( + exclusive_file_lock, + secure_file_descriptor_permissions, +) +from agent.redact import redact_sensitive_text + + +class DestinationClass(StrEnum): + """Trust class for an LLM destination.""" + + LOCAL_PROCESS = "local_process" + LOOPBACK = "loopback" + REMOTE = "remote" + UNKNOWN = "unknown" + + +@dataclass(frozen=True, slots=True) +class SourceGrant: + """Immutable authorization for one exact, already-read file slice.""" + + canonical_path: Path + display_path: str + line_start: int + line_end: int + content_sha256: str + byte_count: int + session_id: str + turn_id: str + request_id: str + policy_digest: str + + +@dataclass(frozen=True, slots=True) +class LiteralSegment: + """Application-owned literal text for a typed outbound request.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class SanitizedSegment: + """Non-source text that must still pass final secret and encoding scans.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class GeneratedContextSegment: + """Hermes-generated remote context after unsafe-text redaction.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class CodexReasoningReplaySegment: + """Opaque encrypted reasoning state replayed only to the Codex endpoint.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class GeneratedContextKey: + """Application-owned JSON key for generated provider tool schemas.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class UntrustedProvenanceSegment: + """Content-free marker for tool bytes with no trusted origin proof.""" + + content_sha256: str + + +@dataclass(frozen=True, slots=True) +class ValidatedToolSyntaxSegment: + """Strictly parsed syntax emitted by a protected local tool result.""" + + text: str + syntax_kind: str + + +@dataclass(frozen=True, slots=True) +class SourceBoundSegment: + """Opaque reference whose text is loaded only from a verified grant.""" + + source_grant_digest: str + + +@dataclass(frozen=True, slots=True) +class SourcePresentationSegment: + """Trusted deterministic presentation of one verified source grant.""" + + source_grant_digest: str + text: str + presentation_kind: str + + +@dataclass(frozen=True, slots=True) +class OutboundText: + """Ordered typed segments that construct one outbound JSON string.""" + + segments: tuple[ + LiteralSegment + | SanitizedSegment + | GeneratedContextSegment + | CodexReasoningReplaySegment + | ValidatedToolSyntaxSegment + | SourceBoundSegment + | SourcePresentationSegment + | UntrustedProvenanceSegment, + ..., + ] + + +@dataclass(frozen=True, slots=True) +class TypedOutboundRequest: + """Remote request recipe; no independent raw string leaf is permitted.""" + + payload: Mapping[str, Any] + session_id: str + turn_id: str + request_id: str + policy_digest: str + + +@dataclass(frozen=True, slots=True) +class EgressDecision: + """Content-free result of one egress preflight.""" + + allowed: bool + destination_class: DestinationClass + provider: str + model: str + payload_sha256: str + serialized_bytes: int + estimated_tokens: int + source_grant_count: int + source_segment_count: int + session_id: str + turn_id: str + request_id: str + policy_digest: str + reason_codes: tuple[str, ...] = () + base_url: str = "" + api_mode: str = "" + grant_digests: tuple[str, ...] = () + + +class EgressBlocked(RuntimeError): + """Raised when the final request is not authorized for its destination.""" + + def __init__(self, decision: EgressDecision): + self.decision = decision + reasons = ",".join(decision.reason_codes) or "policy_denied" + super().__init__(f"LLM egress blocked: {reasons}") + + +class SanitizedTextRejected(ValueError): + """Raised when remote text cannot earn the sanitized segment type.""" + + def __init__(self, reason_code: str): + self.reason_code = reason_code + super().__init__(f"sanitized remote text rejected: {reason_code}") + + +@dataclass(frozen=True, slots=True) +class AuthorizedEgress: + """Immutable exact bytes that a provider callback is authorized to send.""" + + decision: EgressDecision + payload_bytes: bytes + + @property + def allowed(self) -> bool: + return self.decision.allowed + + @property + def destination_class(self) -> DestinationClass: + return self.decision.destination_class + + @property + def reason_codes(self) -> tuple[str, ...]: + return self.decision.reason_codes + + def verify_payload(self, candidate: bytes) -> bytes: + """Return the authorized bytes or reject a post-preflight mutation.""" + + if not isinstance(candidate, bytes) or not compare_digest( + sha256(candidate).hexdigest(), + self.decision.payload_sha256, + ): + raise EgressBlocked( + replace( + self.decision, + allowed=False, + reason_codes=("payload_digest_mismatch",), + ) + ) + return self.payload_bytes + + +_SAFE_ID = re.compile(r"^[A-Za-z0-9_.:@/-]{0,256}$") +_LOCAL_PROCESS_MODES = frozenset({"local_process", "in_process"}) +_BASE64_CANDIDATE = re.compile( + r"(?--[a-z][a-z0-9]*(?:-[a-z0-9]+)*=)" + r"(?P[A-Z]{3,8})(?P[^A-Za-z0-9_+/=-])" +) +_BOUNDED_SOURCE_REVIEW_SYNTAX = re.compile( + r"(?:(?<=--event )(?:APPROVE|REQUEST_CHANGES|COMMENT)(?=[|` ])" + r"|(?<=\|)(?:APPROVE|REQUEST_CHANGES|COMMENT)(?=[|` ])" + r"|(?<=--body )TEXT(?=[`;]|\Z))" +) +_BOUNDED_SOURCE_ADVISORY_KEY = re.compile( + r"(?:GHSA-[A-Za-z0-9]+(?:-[A-Za-z0-9]+){2,8}|PYSEC-[0-9]{4}-[0-9]{3,})" +) +_BOUNDED_SOURCE_GIT_HEAD_OUTPUT = re.compile( + r"(?m)^(?P[0-9a-f]{40})\n(?P[0-9a-f]{10})[0-9a-f]{0,30} [^\n]*$" +) +_BOUNDED_SOURCE_GIT_LOG_ENTRY = re.compile( + r"(?m)^(?P[0-9a-f]{40})(?= " + r"(?:fix|feat|test|docs|chore|refactor|perf|build|ci|style|revert|Merge)" + r"(?:\(|:|\s))" +) +_BOUNDED_NUMBERED_SOURCE_GIT_LOG_ENTRY = re.compile( + r"(?m)^\d+\|[0-9a-f]{7,12}\s+" + r"(?=(?:fix|feat|test|docs|chore|refactor|perf|build|ci|style|revert|Merge)" + r"(?:\(|:|\s))[^\n]*$" +) +_BOUNDED_SOURCE_DIFF_STAT_BINARY = re.compile( + r"(?m)^(?:\d+\|\s*)?(?P[^\n|]*\|\s+Bin\s+\d+\s*->\s+)" + r"(?P\d+)(?P\s+bytes\s*)$" +) +_BOUNDED_SOURCE_DIFF_STAT_COUNT = re.compile( + r"(?m)^(?:\d+\|\s*)?(?P[^\n|]*\|\s+)(?P\d{1,8})" + r"(?P\s+[+]+\s*)$" +) +_BOUNDED_SOURCE_NUMSTAT_PATH = re.compile( + r"(?m)^(?P\d+\t\d+\t)(?P" + r"(?:[A-Za-z0-9_.-]+/)+[A-Za-z0-9_.+@-]+(?:\.[A-Za-z0-9_.-]+)?" + r"|[A-Za-z0-9_.+@-]+\.[A-Za-z0-9_.-]+)\s*$" +) +_BOUNDED_SOURCE_PATH_FRAGMENT = re.compile( + r"^[A-Za-z0-9]/[A-Za-z0-9._-]{2,}(?:/[A-Za-z0-9._-]{2,})+$" +) +# Protected repair workers persist a bare full Git SHA in ``current_head.txt`` +# and prefix porcelain paths in ``changed_names.txt``. These are bounded +# source-presentation receipts, not encoded payloads. +_BOUNDED_SOURCE_RECEIPT_SHA = re.compile(r"^[0-9a-f]{40}$") +_BOUNDED_SOURCE_CHANGED_NAME = re.compile( + r"^zz_changed__[A-Za-z0-9][A-Za-z0-9._-]{2,191}$" +) +_BOUNDED_SOURCE_DASHED_TITLE = re.compile( + r"^[A-Z][A-Za-z0-9]+(?:-[A-Za-z][A-Za-z0-9]+)+$" +) +_BOUNDED_SOURCE_LINE_LABEL = re.compile(r"^[nN][0-9]{2,6}$") +_BOUNDED_SOURCE_FILE_TOKEN = re.compile( + r"\b[A-Za-z][A-Za-z0-9]*(?:_[A-Za-z][A-Za-z0-9]*)+\.[A-Za-z0-9]{1,8}\b" +) +_BOUNDED_ISO_DURATION = re.compile(r"\bP[0-9]{1,4}[DWMY]\b") +_BOUNDED_SOURCE_CODE_ASSIGNMENT = re.compile( + r"\b[a-z][a-z0-9]*(?:_[a-z0-9]+){1,7}=" +) +_BOUNDED_SOURCE_ISSUE_KEY = re.compile( + r"\b[A-Z]{1,8}\d{2,}-[A-Z0-9]+(?:-[A-Z0-9]+){1,8}\b" +) +_BOUNDED_SOURCE_DIFF_METADATA = re.compile( + r"(?m)^\+[A-Za-z0-9+/=_-]{1,128}\s*$" +) +_BOUNDED_SOURCE_DIFF_HUNK = re.compile( + r"(?m)^(?:\d+\|)?@@ -\d{1,8}(?:,\d{1,8})? \+\d{1,8}(?:,\d{1,8})? @@[^\n]*$" +) +_BOUNDED_NUMBERED_SOURCE_DIFF_LINE = re.compile( + r"(?m)^(?:(?P\d+)\|)?(?P[+-])(?P[^\n]*)$" +) +_BOUNDED_SOURCE_SECRET_NAMED_CODE_ASSIGNMENT = re.compile( + r"\b(?P[a-z][a-z0-9_]*_(?:pass|token|secret|password|auth|key))" + r"\s*=\s*(?P_[a-z][a-z0-9_]*(?:\([^\n]*\))?|[a-z][a-z0-9_]*)" +) +_BOUNDED_SOURCE_SECRET_CODE_ASSIGNMENT = re.compile( + r"\b(?P(?:token|secret|password|passwd|api[_-]?key|apikey|" + r"client[_-]?secret|private[_-]?key|(?:[a-z][a-z0-9_]*_)?credentials))" + r"\s*=\s*(?P_[A-Za-z][A-Za-z0-9_]*(?:\([^\n]*\))?|" + r"[A-Za-z][A-Za-z0-9_]*(?:\([^\n]*\))?)(?![A-Za-z0-9_-])" +) +_BOUNDED_SOURCE_NUMERIC_CONSTANT_ASSIGNMENT = re.compile( + r"\b[A-Z][A-Z0-9_]{2,63}\s*=\s*[0-9][0-9_]{2,63}\b" +) +_BOUNDED_SOURCE_SECRET_PLACEHOLDER = re.compile( + r"(?i)\b(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token|secret|" + r"password|passwd|api[_-]?key|apikey|client[_-]?secret|private[_-]?key)" + r"\s*[:=]\s*[\"']?(?:stale-key|legacy-stale-key|test-key|dummy-key|" + r"example-key|placeholder-key|redacted-key)[\"']?\b" +) +_BOUNDED_SOURCE_SECRET_PLACEHOLDER_VALUE = re.compile( + r"(?P[:=]\s*)(?:" + r"ghp_x{4,}|xox[bap]-\.\.\.|your_[a-z0-9_]+|" + r"x{4,}(?:\s+x{4,})*|\*{3,}" + r")(?=$|[\s#])", + re.IGNORECASE, +) +_BOUNDED_SOURCE_SECRET_ENV_NAME = re.compile( + r"\b[A-Z][A-Z0-9_]{2,63}_(?:TOKEN|SECRET|PASSWORD|PASSWD|" + r"API_KEY|PRIVATE_KEY|CREDENTIALS)\b" +) +_BOUNDED_SOURCE_UPPER_SECRET_CODE_ASSIGNMENT = re.compile( + r"(?)[^\s,}\"']+" +) +# These are fixed provider-protocol grammar atoms, not a caller-configurable +# egress allowlist. Several happen to round-trip as unpadded Base64 even though +# they are required JSON schema words. They still go through the final secret +# scan; this set only resolves the mathematical ambiguity in Base64 detection. +_PROTOCOL_GRAMMAR_ATOMS = frozenset( + { + "--noEmit", + "--repository", + "--result", + "-removed", + "100K", + "2000", + "2026", + "4dae", + "600s", + "BOTH", + "COVERAGE", + "EPUB", + "FTS5", + "FULL", + "GGUF", + "MMLU", + "OPTIONAL", + "RELATIVE", + "REPLACES", + "REST", + "SKIP", + "THAT", + "TODO", + "UNAVAILABLE", + "WAIT", + "WHEN", + "HERMES_KANBAN_DB", + "HERMES_KANBAN_BRANCH", + "HERMES_KANBAN_CLAIM_LOCK", + "HERMES_KANBAN_RUN_ID", + "HERMES_KANBAN_WORKSPACE", + "HERMES_CONTROL_HOME", + "HERMES_HOME=", + "HERMES_SESSION_ID", + "HERMES_STREAM_STALE_GIVEUP", + "HERMES_TURN_LEASE_TIMEOUT", + "HEAD", + "HTTP", + "HYGIENE", + "LAST", + "MESSAGE", + "MIME", + "MODE", + "MUTUALLY", + "MUST", + "NOTE", + "ONLY", + "PARALLEL", + "PATH", + "PRAGMA", + "REPL", + "REPLACE", + "REQUIRED", + "SILENTLY", + "THIS", + "USER", + "WSL1", + "_is_git_worktree", + "assistant", + "already-resolved", + "assignee/profile", + "computer_call_output", + "claim/finalize/retry", + "com/docs", + "content", + "codex_review_request", + "current_step_key", + "developer", + "doc/", + "dispatcher_current_directory", + "evidence_heading", + "acceptance-valid", + "architecture-diagram", + "autonomous-ai-agents", + "available_skills", + "background-first", + "document-to-action-items", + "evaluating-llms-harness", + "filesystem-writing", + "find_referencing_symbols", + "get_symbols_overview", + "github-pr-workflow", + "google-workspace", + "hermes_independent_code_review", + "hermes-agent-skill-authoring", + "match_message_id", + "meeting-action-items", + "merge-reconciler", + "p5js", + "popular-web-designs", + "requesting-code-review", + "software-development", + "songwriting-and-ai-music", + "systematic-debugging", + "weekly-review-planning", + "50KB", + "1800", + "8787", + "ids/goals/status/transcripts", + "references/templates/scripts", + "echo/cat", + "echo/heredoc", + "environment-variable", + "find-and-replace", + "function_call", + "function_call_output", + "+for", + "repeated_exact_failure_block", + "_force_close_actionable_pending_routes_for_cycle", + "github-code-review", + "grep/rg/find/ls", + "include_archived", + "input_image", + "input_text", + "JSON", + "kanban_heartbeat", + "machine-readable", + "max_runtime_seconds", + "messages", + "n_error_", + "notification/15s", + "optional-profile", + "OPEN/MERGEABLE/CLEAN", + "output_text", + "parent/child", + "parents=", + "parallel_tool_calls", + "path/to/file", + "ppt/", + "prepare_receipt_worktree", + "prompt_cache_key", + "protected-remote", + "repository-owned", + "runtime-executed", + "reasoning", + "reasoning_effort", + "role", + "sed/awk", + "servers/daemons", + "servers/watchers/daemons", + "session_resolver", + "skills/plugins/cron/memories", + "logic-regression", + "system", + "tool", + "untrusted_provenance", + "user", + "workspace_access", + "workflow_template_id", + } +) + + +def classify_destination( + provider: str, + base_url: str | None, + api_mode: str | None, +) -> DestinationClass: + """Classify without DNS, provider-name, or private-network trust. + + Only an explicit in-process mode or a numeric loopback literal is local. + LAN, Tailscale, container DNS, ``localhost``, and other hostnames retain + remote policy. Missing or malformed endpoint data is unknown. + """ + + del provider # Provider labels are not a security boundary. + mode = str(api_mode or "").strip().lower() + if mode in _LOCAL_PROCESS_MODES: + return DestinationClass.LOCAL_PROCESS + if not isinstance(base_url, str) or not base_url.strip(): + return DestinationClass.UNKNOWN + try: + parsed = urlsplit(base_url.strip()) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + return DestinationClass.UNKNOWN + # ``hostname`` does not validate the port. Accessing ``port`` is + # load-bearing: urllib deliberately raises for non-numeric and + # out-of-range values that must never inherit loopback trust. + try: + parsed.port + except ValueError: + return DestinationClass.UNKNOWN + if parsed.netloc.rsplit("@", 1)[-1].endswith(":"): + return DestinationClass.UNKNOWN + host = parsed.hostname + if not host: + return DestinationClass.UNKNOWN + try: + address = ipaddress.ip_address(host) + except ValueError: + return DestinationClass.REMOTE + return DestinationClass.LOOPBACK if address.is_loopback else DestinationClass.REMOTE + except (TypeError, ValueError): + return DestinationClass.UNKNOWN + + +def _route_value(route: Any, name: str, default: Any = None) -> Any: + if isinstance(route, Mapping): + return route.get(name, default) + return getattr(route, name, default) + + +def _request_identity(request: Mapping[str, Any], name: str) -> str: + value = request.get(name, "") + return value if isinstance(value, str) else str(value) + + +def _receipt_identifier(value: str) -> str: + """Keep ordinary correlation IDs while hashing unsafe or sensitive labels.""" + + try: + safe = ( + _SAFE_ID.fullmatch(value) is not None + and not value.startswith(("/", "~")) + and redact_sensitive_text( + value, + force=True, + redact_url_credentials=True, + ) + == value + ) + except Exception: + safe = False + return value if safe else f"sha256:{sha256(value.encode('utf-8')).hexdigest()}" + + +def source_grant_digest(grant: SourceGrant) -> str: + """Return the opaque identity used by a source-segment manifest.""" + + bound_fields = { + "canonical_path": str(grant.canonical_path), + "display_path": grant.display_path, + "line_start": grant.line_start, + "line_end": grant.line_end, + "content_sha256": grant.content_sha256, + "byte_count": grant.byte_count, + "session_id": grant.session_id, + "turn_id": grant.turn_id, + "request_id": grant.request_id, + "policy_digest": grant.policy_digest, + } + encoded = json.dumps(bound_fields, separators=(",", ":"), sort_keys=True).encode("utf-8") + return sha256(encoded).hexdigest() + + +def static_literal_sha256(text: str) -> str: + """Hash one exact UTF-8 static literal for a policy allowlist.""" + + if not isinstance(text, str): + raise TypeError("static literal must be text") + return sha256(text.encode("utf-8")).hexdigest() + + +def validate_tool_syntax(text: str, syntax_kind: str) -> str: + """Revalidate one complete protected tool-result syntax atom.""" + + grammar = _VALIDATED_TOOL_SYNTAX.get(syntax_kind) + if not isinstance(text, str) or grammar is None or grammar.fullmatch(text) is None: + raise ValueError("invalid_tool_syntax_segment") + if _contains_secret(text) or _contains_private_absolute_path(text): + raise ValueError("invalid_tool_syntax_segment") + return text + + +def _canonical_base64_candidate(candidate: str) -> bool: + """Recognize bounded canonical encodings without flagging ordinary IDs.""" + + # Source-diff callers supply whole lines, including Unicode comments. + # Neither standard nor URL-safe Base64 has non-ASCII alphabet members. + if not candidate.isascii(): + return False + if candidate in _PROTOCOL_GRAMMAR_ATOMS: + return False + if candidate in _SAFE_DIAGNOSTIC_STATUS_WORDS: + # Exact status labels are not an encoding channel; treating them as + # Base64 strands workers while replaying ordinary CLI output. + return False + if _LINTER_DIAGNOSTIC_CODE.fullmatch(candidate): + # Ruff/flake8-style findings are ordinary bounded CI metadata. They + # are not source excerpts or opaque encoded payloads, even when a + # tool result is carried as a SanitizedSegment rather than generated + # context where the source-atom mask would already apply. + return False + if _PYTHON_DUNDER_IDENTIFIER.fullmatch(candidate): + # Python's bounded dunder names are source-language structure, not + # encoded content (for example __file__ and __main__ in CI scripts). + return False + if ( + _BOUNDED_VERSIONED_IDENTIFIER.fullmatch(candidate) + or _BOUNDED_TEST_ARTIFACT.fullmatch(candidate) + or _BOUNDED_FUNCTION_IDENTIFIER.fullmatch(candidate) + ): + # Filenames, test names, model slugs, and config identifiers are + # normal generated/tool context. They are not encoded content merely + # because their lexical shape happens to decode canonically. + return False + if ( + _BOUNDED_SHORT_CLI_OPTION.fullmatch(candidate) + or _BOUNDED_LINE_RANGE_OR_UNIT.fullmatch(candidate) + or _BOUNDED_STATUS_COUNT.fullmatch(candidate) + or _BOUNDED_COMMAND_PATH.fullmatch(candidate) + or _BOUNDED_RENDER_MARKER.fullmatch(candidate) + ): + return False + if _BOUNDED_SLASH_WORDS.fullmatch(candidate): + # Bounded relative paths such as venv/lib/python3 are ordinary local + # CI metadata, not encoded content. Keep the grammar narrow so an + # unrecognized underscore atom remains fail-closed below. + return False + if _BOUNDED_DURATION.fullmatch(candidate): + return False + if re.fullmatch(r"0x[0-9a-fA-F]+", candidate): + return False + if not 4 <= len(candidate) <= _MAX_BASE64_CANDIDATE_CHARS: + return False + # Short words and word-shaped structural fragments frequently round-trip + # mathematically as unpadded Base64. Their bounded lexical form is the + # disambiguating signal; padding, digits, mixed punctuation, and long + # ambiguous blobs remain eligible for canonical decoding below. + if len(candidate) < 24: + if candidate.isalpha() and not candidate.isupper(): + return False + if _BOUNDED_CLI_WORD.fullmatch(candidate): + return False + if _BOUNDED_SLASH_WORDS.fullmatch(candidate): + return False + # Short, word-like URL-safe slugs are common model/provider identifiers. + # Keep genuinely encoding-shaped values such as ``-_8A`` eligible for the + # canonical decoder below. + if ( + len(candidate) < 16 + and any(character in "-_" for character in candidate) + and candidate[0].isalnum() + and candidate[-1].isalnum() + and sum(character.isalpha() for character in candidate) >= 2 + ): + return False + unpadded = candidate.rstrip("=") + if "=" in unpadded or len(unpadded) % 4 == 1: + return False + padded = unpadded + "=" * (-len(unpadded) % 4) + encoded = padded.encode("ascii") + for altchars in (None, b"-_"): + try: + decoded = base64.b64decode(encoded, altchars=altchars, validate=True) + except (binascii.Error, ValueError): + continue + canonical_padded = ( + base64.b64encode(decoded) + if altchars is None + else base64.urlsafe_b64encode(decoded) + ).decode("ascii") + if candidate in {canonical_padded, canonical_padded.rstrip("=")} and decoded: + return True + return False + + +def _canonical_chunked_base64_candidate(candidate: str) -> bool: + """Recognize fixed-width wrapped encodings without joining ordinary prose.""" + + chunks = re.findall(r"[A-Za-z0-9_+/=-]{2,4}", candidate) + if len(chunks) < 3: + return False + if all(_LINTER_DIAGNOSTIC_CODE.fullmatch(chunk) for chunk in chunks): + # A run of bounded Ruff/flake8 findings is structured CI output, not + # a wrapped encoding. Without this guard, ``E501 F821 W391`` is joined + # across spaces and can happen to decode canonically. + return False + width = len(chunks[0]) + if not all(len(chunk) == width for chunk in chunks[:-1]): + return False + if len(chunks[-1]) > width: + return False + joined = "".join(chunks) + has_encoding_signal = any( + character.isupper() or character.isdigit() or character in "+/_=" + for character in joined + ) + if len(chunks) < 8 and not has_encoding_signal: + return False + return _canonical_base64_candidate(joined) + + +# GitHub's legacy GraphQL global node id: base64 of a fixed +# ``:`` grammar (e.g. "05:Issue160502814" -> +# "MDU6SXNzdWUxNjA1MDI4MTQ0"). `gh api` / `gh issue|pr list` return these in +# every issue/PR/label/comment object's `node_id` field, so any tool-output +# scan that fetches GitHub issue or PR data structurally contains them. They +# are fixed, low-entropy protocol identifiers for public object identity — +# not caller-supplied encoded content — so they get the same treatment as +# the other bounded protocol grammars above (call_/fc_ ids, kanban task ids, +# prompt cache keys). Investigation for t_80e6f80b: this pattern was +# repeatedly and falsely flagged as ``base64_payload`` on every provider +# fallback from a local model to a REMOTE one mid-scan, permanently +# excluding cloud fallback for any GitHub-issue-reading profile. +_GITHUB_LEGACY_NODE_ID_GRAMMAR = re.compile(r"\A\d{1,3}:[A-Za-z]{2,40}\d{1,20}\Z") + + +def _looks_like_github_legacy_node_id(candidate: str) -> bool: + unpadded = candidate.rstrip("=") + if "=" in unpadded or len(unpadded) % 4 == 1: + return False + padded = unpadded + "=" * (-len(unpadded) % 4) + try: + decoded = base64.b64decode(padded.encode("ascii"), validate=True) + except (binascii.Error, ValueError): + return False + try: + text = decoded.decode("ascii") + except UnicodeDecodeError: + return False + return bool(_GITHUB_LEGACY_NODE_ID_GRAMMAR.fullmatch(text)) + + +def _contains_canonical_base64(value: Any, *, seen: set[int] | None = None) -> bool: + if isinstance(value, str): + # Fixed Hermes/Nous attribution tags are protocol metadata, not an + # encoded source payload. They remain subject to secret/path scans. + if value.startswith(("product=hermes-agent", "client=hermes-client-")): + return False + for match in _BASE64_CANDIDATE.finditer(value): + candidate = match.group(1) + prefix = value[max(0, match.start() - 16) : match.start()].lower() + source_control_window = value[ + max(0, match.start() - 48) : min(len(value), match.end() + 16) + ] + if re.fullmatch( + r"[0-9a-f]{7,12}|[0-9a-f]{40}|[0-9a-f]{64}", + candidate.lower(), + ): + continue + if ( + _BOUNDED_SOURCE_CONTROL_FRAGMENT.fullmatch(candidate.lower()) + and _SOURCE_CONTROL_CONTEXT.search(source_control_window) + ): + # Shortened git object IDs are ordinary source-control + # metadata when explicitly labeled as such. Without that + # context, arbitrary hex remains fail-closed. + continue + if candidate.isdigit(): + before = value[: match.start(1)].rstrip()[-1:] + after = value[match.end(1) :].lstrip()[:1] + if before in {":", ",", "["} and after in {",", "]", "}"}: + # JSON numeric values in tool results are serialized + # protocol fields, not encoded text. A quoted numeric + # string remains eligible for Base64 detection. + continue + # The fixed Kanban task-id grammar carries only a 32-bit hex + # database key. It is application protocol metadata, not an + # encoded source payload. + if _HERMES_TASK_ID.fullmatch(candidate): + continue + # Provider-generated tool-call and response-item identifiers are + # opaque protocol routing metadata, not caller-supplied encoded + # content. Match their complete, fixed grammar only. + if re.fullmatch(r"(?:call|fc)_[A-Za-z0-9_-]{8,128}", candidate): + continue + if candidate in { + "HERMES_CONTROL_HOME", + "HERMES_KANBAN_DB", + "HERMES_KANBAN_WORKSPACES_ROOT", + "HERMES_PROFILE_HOME", + }: + continue + # Content-addressed cache routing is a fixed application protocol + # value: the literal ``pck_`` prefix plus exactly 96 bits of hex. + if _PROMPT_CACHE_KEY.fullmatch(candidate): + continue + # GitHub's legacy global node id (see helper docstring above). + if _looks_like_github_legacy_node_id(candidate): + continue + if _canonical_base64_candidate(candidate): + return True + # Providers and source-control tools sometimes wrap an otherwise + # canonical encoding at a fixed column. Normalize only bounded chunks + # so ordinary prose words are not concatenated into a false candidate. + for match in _CHUNKED_BASE64_CANDIDATE.finditer(value): + if _canonical_chunked_base64_candidate(match.group(0)): + return True + return False + if isinstance(value, (bytes, bytearray, memoryview)): + return True + if seen is None: + seen = set() + if isinstance(value, Mapping): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + # Typed request keys are application-owned structure, not payload + # segments. Only values can carry caller-controlled concealment. + return any(_contains_canonical_base64(item, seen=seen) for item in value.values()) + if isinstance(value, (list, tuple, set, frozenset)): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any(_contains_canonical_base64(item, seen=seen) for item in value) + return False + + +def _source_text_for_base64_scan( + text: str, *, allow_fixed_source_literals: bool = False +) -> str: + """Mask bounded code atoms only after exact source-grant validation. + + Snake-case config keys, lowercase kebab-case rule names, and linter codes + can mathematically round-trip as unpadded Base64. Their grammar is + low-entropy and ordinary in source files. Actual encoded blobs remain + unchanged and are still rejected by the canonical scanner. + """ + + def is_source_code_atom(match: re.Match[str], source_text: str) -> bool: + # ``_BASE64_CANDIDATE`` includes padding characters in the match, so + # a source keyword such as ``line_ranges=`` arrives here with its + # trailing assignment marker attached. Strip only that marker; a + # padded encoded value remains unchanged and fail-closed. + candidate = match.group(1) + source_atom = candidate[:-1] if candidate.endswith("=") else candidate + source_control_window = source_text[ + max(0, match.start() - 256) : min(len(source_text), match.end() + 32) + ] + line_start = source_text.rfind("\n", 0, match.start()) + 1 + line_end = source_text.find("\n", match.end()) + if line_end < 0: + line_end = len(source_text) + source_line = source_text[line_start:line_end] + is_numbered_receipt_sha = ( + allow_fixed_source_literals + and _BOUNDED_SOURCE_RECEIPT_SHA.fullmatch(source_atom) is not None + and re.fullmatch(r"\s*\d+\|[0-9a-f]{40}\s*", source_line) is not None + ) + is_source_control_identity = ( + re.fullmatch( + r"[0-9a-f]{7,12}|[0-9a-f]{40}|[0-9a-f]{64}", + candidate.lower(), + ) + is not None + and ( + _SOURCE_CONTROL_CONTEXT.search(source_control_window) is not None + or _GITHUB_SOURCE_CONTROL_URL_CONTEXT.search(source_control_window) + is not None + ) + ) + return ( + is_source_control_identity + or + _BOUNDED_SOURCE_CODE_ATOM.fullmatch(source_atom) is not None + or _BOUNDED_LONG_SOURCE_CODE_ATOM.fullmatch(source_atom) is not None + or _BOUNDED_LONG_PRIVATE_IDENTIFIER.fullmatch(source_atom) is not None + or _BOUNDED_SOURCE_DOUBLE_UNDERSCORE.fullmatch(source_atom) is not None + or _BOUNDED_SOURCE_VERSIONED_IDENTIFIER.fullmatch(source_atom) is not None + or _PYTHON_DUNDER_IDENTIFIER.fullmatch(source_atom) is not None + or _PYTHON_PRIVATE_IDENTIFIER.fullmatch(source_atom) is not None + or _PYTHON_MIXED_CASE_IDENTIFIER.fullmatch(source_atom) is not None + or _BOUNDED_PASCAL_CASE_IDENTIFIER.fullmatch(source_atom) is not None + or _BOUNDED_SOURCE_CAMEL_CASE_IDENTIFIER.fullmatch(source_atom) is not None + or _BOUNDED_SOURCE_ADVISORY_KEY.fullmatch(source_atom) is not None + or _BOUNDED_SOURCE_PATH_FRAGMENT.fullmatch(source_atom) is not None + or is_numbered_receipt_sha + or ( + allow_fixed_source_literals + and _BOUNDED_SOURCE_CHANGED_NAME.fullmatch(source_atom) is not None + ) + or _BOUNDED_SOURCE_DASHED_TITLE.fullmatch(source_atom) is not None + or _BOUNDED_SOURCE_LINE_LABEL.fullmatch(source_atom) is not None + or ( + allow_fixed_source_literals + and source_atom in _BOUNDED_SOURCE_FIXED_LITERAL_ATOMS + ) + or source_atom in { + "LICENSE", + "BM25", + "HTML", + "PKCS", + "IANA", + "CONTRIBUTING", + "sprmn24", + "BOUNDARY", + "CASH", + "FIFO", + "FIRE", + "MPLCONFIGDIR", + } + # argparse usage renders a small, fixed set of all-caps + # metavariables. They are command syntax, not encoded payloads; + # keep this exception enumerated so arbitrary values such as + # ``PAYLOAD`` remain visible to the fail-closed scanner. + # SQL snippets in source comments/queries likewise use fixed + # keywords whose short uppercase spelling can decode by chance. + or source_atom in {"PROVIDER", "TOOLSETS", "OPEN", "LIKE", "YAML"} + ) + + def is_source_identifier_in_code( + match: re.Match[str], source_text: str + ) -> bool: + candidate = match.group(1) + source_atom = candidate[:-1] if candidate.endswith("=") else candidate + if re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{2,191}", source_atom) is None: + return False + line_start = source_text.rfind("\n", 0, match.start()) + 1 + line_end = source_text.find("\n", match.end()) + if line_end < 0: + line_end = len(source_text) + line = source_text[line_start:line_end] + offset = match.start() - line_start + before = line[:offset] + after = line[match.end() - line_start :] + if before.rstrip().endswith(("'", '"')): + return False + if re.search(r"\b(?:def|class)\s+$", before): + return True + at_line_assignment = not before and re.match(r"\s*=", after) is not None + return ( + not before.rstrip().endswith((".", "'", '"')) + and ( + at_line_assignment + or ( + bool(before) + and before[-1] in " \t([{,:;" + ) + ) + and re.match(r"\s*(?:[=,.;:)(\]}])", after) is not None + ) + + # Explicit credential placeholders in checked-in examples are source + # grammar, not credential material. Keep the exception exact and scoped + # to validated source grants; arbitrary credential-shaped values remain + # visible to the fail-closed scan. + masked = _BOUNDED_SOURCE_SECRET_PLACEHOLDER_VALUE.sub( + r"\g", text + ) + # A source path such as ``execution_submit_boundary.py`` is one lexical + # token, but the Base64 candidate regex sees its uppercase suffix after + # the underscore as a standalone candidate. Mask the whole path-shaped + # token before that scan; an opaque payload cannot contain a dot. + masked = _BOUNDED_SOURCE_FILE_TOKEN.sub("", masked) + masked = _BOUNDED_ISO_DURATION.sub("", masked) + masked = _BOUNDED_SOURCE_NUMERIC_CONSTANT_ASSIGNMENT.sub( + "", masked + ) + masked = _BOUNDED_NUMBERED_SOURCE_GIT_LOG_ENTRY.sub( + "", masked + ) + masked = _BOUNDED_SOURCE_DIFF_STAT_BINARY.sub( + "", masked + ) + masked = _BOUNDED_SOURCE_DIFF_STAT_COUNT.sub( + "", masked + ) + masked = _BASE64_CANDIDATE.sub( + lambda match: ( + "" + if is_source_code_atom(match, masked) + or is_source_identifier_in_code(match, masked) + else match.group(0) + ), + masked, + ) + # A simple assignment such as ``sources=sources`` can expose the left + # hand side after the right-hand identifier is replaced above. Mask the + # fixed source keys as a whole so the second scan cannot manufacture a + # new ``sources=`` Base64 candidate at that boundary. + if allow_fixed_source_literals: + masked = re.sub( + r"\b(?:columns|reasons|sources|targets)=", + "", + masked, + ) + # CLI filter values such as ``--diff-filter=ACMR`` are source syntax, not + # opaque payloads. Keep this grammar tied to a long-option assignment so + # short quoted Base64 values elsewhere remain rejected. + masked = _BOUNDED_SOURCE_CODE_ASSIGNMENT.sub("", masked) + masked = _BOUNDED_SOURCE_ISSUE_KEY.sub("", masked) + + def mask_diff_metadata(match: re.Match[str]) -> str: + line = match.group(0) + candidate = line[1:].strip() + # A unified-diff marker is metadata only when the added line itself + # is not an encoded payload. Keep canonical Base64 (including wrapped + # form) visible to the fail-closed scanner. + if _canonical_base64_candidate(candidate) or _canonical_chunked_base64_candidate( + candidate + ): + # Separate the marker from the candidate. The candidate regex + # includes ``+`` in its URL-safe alphabet, so returning ``+blob`` + # would change the bytes being tested and accidentally hide a + # real encoded payload behind the diff marker. + suffix = "\n" if line.endswith("\n") else "" + return "+ " + candidate + suffix + return "" + + masked = _BOUNDED_SOURCE_DIFF_METADATA.sub(mask_diff_metadata, masked) + masked = _BOUNDED_SOURCE_DIFF_HUNK.sub("", masked) + + def mask_numbered_diff_line(match: re.Match[str]) -> str: + body = match.group("body").strip() + candidate = f"{match.group('marker')}{body}" + # Keep an entire added/removed line visible when it is itself an + # encoded payload. Ordinary source syntax (``+def ...``, imports, + # assertions, and so on) is presentation metadata for this scan; the + # independent secret scan still sees the original source bytes. + if ( + _canonical_base64_candidate(candidate) + or _canonical_chunked_base64_candidate(candidate) + or _canonical_base64_candidate(body) + or _canonical_chunked_base64_candidate(body) + ): + return match.group(0) + return "" + + masked = _BOUNDED_NUMBERED_SOURCE_DIFF_LINE.sub(mask_numbered_diff_line, masked) + masked = _BOUNDED_SOURCE_GIT_LOG_ENTRY.sub("", masked) + masked = _BOUNDED_SOURCE_GIT_HEAD_OUTPUT.sub( + lambda match: f"\n {match.group(0).split(' ', 1)[1]}", + masked, + ) + masked = _BOUNDED_SOURCE_DIFF_STAT_BINARY.sub( + "", + masked, + ) + masked = _BOUNDED_SOURCE_DIFF_STAT_COUNT.sub( + "", + masked, + ) + masked = _BOUNDED_SOURCE_NUMSTAT_PATH.sub( + "", + masked, + ) + masked = _BOUNDED_SOURCE_CLI_VALUE.sub( + lambda match: f"{match.group('prefix')}{match.group('suffix')}", + masked, + ) + return _BOUNDED_SOURCE_REVIEW_SYNTAX.sub("", masked) + + +def _generated_context_text_for_base64_scan(text: str) -> str: + """Mask bounded application atoms in already-redacted generated context. + + Generated context is produced by Hermes and has already passed the + secret/path/base64 redaction step. Its ordinary function names, rule + names, and schema identifiers still need the source-style lexical mask at + the final scan; arbitrary encoded values remain untouched and fail closed. + """ + + # A protected Kanban assignment may contain this fixed worker result + # marker. It is application protocol text, not an encoded payload, but its + # all-caps/underscore spelling is a valid Base64 candidate. Keep the + # marker visible on the wire and mask it only in the generated-context + # scan; arbitrary task markers remain fail-closed. + text = text.replace("PAPER_SAFETY_SENTINEL_OK", "") + return _source_text_for_base64_scan(text) + + +def _source_text_for_secret_scan(text: str) -> str: + """Mask code identifiers that resemble secret names, not secret values.""" + + if _EGRESS_SECRET_ASSIGNMENT.search(text) is not None: + text = mask_builtin_annotations(text) + text = _BOUNDED_SOURCE_BOOLEAN_SECRET_SETTING.sub( + "", text + ) + text = _BOUNDED_SOURCE_SECRET_PLACEHOLDER_VALUE.sub( + r"\g", text + ) + text = _BOUNDED_SOURCE_SECRET_EXPRESSION_ASSIGNMENT.sub( + "", text + ) + text = _BOUNDED_SOURCE_SECRET_ENV_NAME.sub("", text) + text = _BOUNDED_SOURCE_UPPER_SECRET_CODE_ASSIGNMENT.sub("", text) + text = _BOUNDED_SOURCE_SECRET_PLACEHOLDER.sub("", text) + text = _BOUNDED_SOURCE_SECRET_CODE_ASSIGNMENT.sub("", text) + return _BOUNDED_SOURCE_SECRET_NAMED_CODE_ASSIGNMENT.sub("", text) + + +def _contains_secret(value: Any, *, seen: set[int] | None = None) -> bool: + """Apply forced redaction semantics independently to every request string.""" + + if isinstance(value, str): + return redact_sensitive_text( + value, + force=True, + redact_url_credentials=True, + ) != value or _EGRESS_SECRET_ASSIGNMENT.search(value) is not None + if isinstance(value, (bytes, bytearray, memoryview)): + # Binary request material is not safely inspectable as text. + return True + if seen is None: + seen = set() + if isinstance(value, Mapping): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any( + _contains_secret(key, seen=seen) or _contains_secret(item, seen=seen) + for key, item in value.items() + ) + if isinstance(value, (list, tuple, set, frozenset)): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any(_contains_secret(item, seen=seen) for item in value) + return False + + +def _contains_exact_secret( + value: Any, + exact_values: Sequence[str], + *, + seen: set[int] | None = None, +) -> bool: + """Match authoritative applied/environment credential bytes exactly.""" + + if isinstance(value, str): + return any(secret in value for secret in exact_values) + if isinstance(value, (bytes, bytearray, memoryview)): + return True + if seen is None: + seen = set() + if isinstance(value, Mapping): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any( + _contains_exact_secret(key, exact_values, seen=seen) + or _contains_exact_secret(item, exact_values, seen=seen) + for key, item in value.items() + ) + if isinstance(value, (list, tuple, set, frozenset)): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any( + _contains_exact_secret(item, exact_values, seen=seen) for item in value + ) + return False + + +def _contains_private_absolute_path(value: Any, *, seen: set[int] | None = None) -> bool: + """Reject common host-private absolute paths without blocking API paths.""" + + if isinstance(value, str): + return _PRIVATE_ABSOLUTE_PATH.search(value) is not None + if isinstance(value, (bytes, bytearray, memoryview)): + return True + if seen is None: + seen = set() + if isinstance(value, Mapping): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any( + _contains_private_absolute_path(key, seen=seen) + or _contains_private_absolute_path(item, seen=seen) + for key, item in value.items() + ) + if isinstance(value, (list, tuple, set, frozenset)): + identity = id(value) + if identity in seen: + return True + seen.add(identity) + return any(_contains_private_absolute_path(item, seen=seen) for item in value) + return False + + +def redact_remote_unsafe_text(text: str) -> str: + """Redact non-secret unsafe text in Hermes-generated remote context. + + Secrets intentionally remain a hard firewall denial. Private paths and + canonical base64-shaped protocol text can be replaced while preserving + the surrounding system/tool instructions needed by remote models. + """ + + if not isinstance(text, str): + raise TypeError("remote context must be text") + + def replace_path(match: re.Match[str]) -> str: + value = match.group(0) + prefix_match = re.match(r"^[\s\"'`(]*", value) + prefix = prefix_match.group(0) if prefix_match else "" + return prefix + "" + + redacted = _PRIVATE_ABSOLUTE_PATH.sub(replace_path, text) + def replace_base64(match: re.Match[str]) -> str: + candidate = match.group(1) + if re.fullmatch( + r"[0-9a-f]{7,12}|[0-9a-f]{40}|[0-9a-f]{64}", candidate.lower() + ): + return match.group(0) + if candidate.isdigit(): + before = redacted[: match.start(1)].rstrip()[-1:] + after = redacted[match.end(1) :].lstrip()[:1] + if before in {":", ",", "["} and after in {",", "]", "}"}: + return match.group(0) + if candidate in _PROTOCOL_GRAMMAR_ATOMS or _HERMES_TASK_ID.fullmatch(candidate): + return match.group(0) + if _BOUNDED_SOURCE_CODE_ATOM.fullmatch(candidate): + return match.group(0) + if ( + _BOUNDED_LONG_SOURCE_CODE_ATOM.fullmatch(candidate) + or _BOUNDED_LONG_PRIVATE_IDENTIFIER.fullmatch(candidate) + or _BOUNDED_SOURCE_DOUBLE_UNDERSCORE.fullmatch(candidate) + or _BOUNDED_SOURCE_VERSIONED_IDENTIFIER.fullmatch(candidate) + or _BOUNDED_SOURCE_CAMEL_CASE_IDENTIFIER.fullmatch(candidate) + ): + return match.group(0) + if re.fullmatch(r"(?:call|fc)_[A-Za-z0-9_-]{8,128}", candidate): + return match.group(0) + if candidate in { + "HERMES_CONTROL_HOME", + "HERMES_KANBAN_DB", + "HERMES_KANBAN_WORKSPACES_ROOT", + "HERMES_PROFILE_HOME", + }: + return match.group(0) + if _PROMPT_CACHE_KEY.fullmatch(candidate) or _canonical_base64_candidate(candidate): + return "" + return match.group(0) + + redacted = _BASE64_CANDIDATE.sub(replace_base64, redacted) + return _CHUNKED_BASE64_CANDIDATE.sub( + lambda match: "" + if _canonical_chunked_base64_candidate(match.group(0)) + else match.group(0), + redacted, + ) + + +def content_free_violation_locations(value: Any) -> tuple[tuple[str, tuple[str, ...]], ...]: + """Return structural indexes and reasons without returning request text.""" + + locations: list[tuple[str, tuple[str, ...]]] = [] + seen: set[int] = set() + + def visit(item: Any, path: str) -> None: + if isinstance(item, str): + reasons: list[str] = [] + if _contains_canonical_base64(item): + reasons.append("base64_payload") + if _contains_private_absolute_path(item): + reasons.append("private_absolute_path") + if reasons: + locations.append((path, tuple(reasons))) + return + if isinstance(item, Mapping): + identity = id(item) + if identity in seen: + locations.append((path, ("cyclic_container",))) + return + seen.add(identity) + for index, (key, child) in enumerate(item.items()): + visit(key, f"{path}.map[{index}].key") + visit(child, f"{path}.map[{index}].value") + return + if isinstance(item, (list, tuple, set, frozenset)): + identity = id(item) + if identity in seen: + locations.append((path, ("cyclic_container",))) + return + seen.add(identity) + for index, child in enumerate(item): + visit(child, f"{path}.sequence[{index}]") + + visit(value, "$") + return tuple(locations) + + +def _contains_grant_substring(grant_content: bytes, candidate: bytes) -> bool: + """Reject source-derived proper substrings in sanitized text. + + Newline-trimmed line grants commonly appear in JSON without their source + line ending. Exact containment catches short excerpts; otherwise require + a 32-byte shared window so ordinary words such as ``checkout`` do not make + unrelated generated context look source-derived. + """ + + if not grant_content or not candidate: + return False + grant_variants = (grant_content, grant_content.rstrip(b"\r\n")) + for variant in grant_variants: + if not variant: + continue + if variant in candidate: + return True + window = min(32, len(candidate), len(variant)) + if window >= 32: + source_windows = { + variant[offset : offset + window] + for offset in range(0, len(variant) - window + 1) + } + if any( + candidate[offset : offset + window] in source_windows + for offset in range(0, len(candidate) - window + 1) + ): + return True + return False + + +def validate_sanitized_text(text: str, *, max_bytes: int = 32_768) -> str: + """Return unchanged bounded remote-safe text or reject it fail-closed. + + This is the only constructor-side admission path for SanitizedSegment. + The firewall repeats the same scans on the final rendered payload. + """ + + if not isinstance(text, str): + raise SanitizedTextRejected("invalid_sanitized_text") + if max_bytes <= 0 or len(text.encode("utf-8")) > max_bytes: + raise SanitizedTextRejected("sanitized_bytes_exceeded") + try: + if _contains_secret(text): + raise SanitizedTextRejected("secret_detected") + except SanitizedTextRejected: + raise + except Exception as exc: + raise SanitizedTextRejected("redaction_failed") from exc + try: + if _contains_canonical_base64(text): + raise SanitizedTextRejected("base64_payload") + except SanitizedTextRejected: + raise + except Exception as exc: + raise SanitizedTextRejected("base64_scan_failed") from exc + try: + if _contains_private_absolute_path(text): + raise SanitizedTextRejected("private_absolute_path") + except SanitizedTextRejected: + raise + except Exception as exc: + raise SanitizedTextRejected("private_path_scan_failed") from exc + return text + + +def _is_strict_sanitized_only_payload( + value: Any, + *, + seen: set[int] | None = None, +) -> tuple[bool, int]: + """Recognize the one grantless remote shape approved by policy. + + Every text leaf must be an explicit :class:`SanitizedSegment`. Raw text, + static literals, source references, binary values, cycles, and unsupported + containers make the entire payload ineligible. The positive count prevents + an empty structural request from acquiring grantless status. + """ + + if isinstance(value, SanitizedSegment): + return isinstance(value.text, str), 1 if isinstance(value.text, str) else 0 + if isinstance(value, GeneratedContextSegment): + return isinstance(value.text, str), 1 if isinstance(value.text, str) else 0 + if isinstance(value, GeneratedContextKey): + return isinstance(value.text, str), 0 + if isinstance(value, CodexReasoningReplaySegment): + return ( + isinstance(value.text, str) + and _CODEX_ENCRYPTED_REASONING_REPLAY.fullmatch(value.text) is not None, + 1 if isinstance(value.text, str) else 0, + ) + if isinstance(value, UntrustedProvenanceSegment): + return False, 0 + if isinstance(value, ValidatedToolSyntaxSegment): + try: + validate_tool_syntax(value.text, value.syntax_kind) + except (TypeError, ValueError): + return False, 0 + return True, 1 + if isinstance(value, LiteralSegment): + return isinstance(value.text, str), 0 + if isinstance(value, SourceBoundSegment): + return False, 0 + if isinstance(value, SourcePresentationSegment): + return False, 0 + if isinstance(value, OutboundText): + if not value.segments: + return False, 0 + count = 0 + for segment in value.segments: + allowed, segment_count = _is_strict_sanitized_only_payload(segment, seen=seen) + if not allowed: + return False, 0 + count += segment_count + return count > 0, count + if value is None or isinstance(value, (bool, int)): + return True, 0 + if isinstance(value, float): + return math.isfinite(value), 0 + if isinstance(value, (str, bytes, bytearray, memoryview, set, frozenset)): + return False, 0 + if seen is None: + seen = set() + if isinstance(value, Mapping): + identity = id(value) + if identity in seen: + return False, 0 + seen.add(identity) + count = 0 + for key, item in value.items(): + if not isinstance(key, (str, GeneratedContextKey)): + return False, 0 + allowed, item_count = _is_strict_sanitized_only_payload(item, seen=seen) + if not allowed: + return False, 0 + count += item_count + return True, count + if isinstance(value, (list, tuple)): + identity = id(value) + if identity in seen: + return False, 0 + seen.add(identity) + count = 0 + for item in value: + allowed, item_count = _is_strict_sanitized_only_payload(item, seen=seen) + if not allowed: + return False, 0 + count += item_count + return True, count + return False, 0 + + +class LLMEgressFirewall: + """Validate a final LLM request and record a content-free receipt.""" + + def __init__( + self, + state_dir: Path | str, + *, + max_serialized_bytes: int = 262_144, + max_sanitized_bytes: int = 32_768, + max_sanitized_segment_bytes: int = 32_768, + max_conservative_tokens: int = 87_382, + max_granted_serialized_bytes: int | None = None, + max_granted_conservative_tokens: int | None = None, + conservative_chars_per_token: int = 3, + policy_digest: str | None = None, + static_literal_hashes_by_policy: Mapping[str, Sequence[str]] | None = None, + exact_secret_values: Sequence[str] = (), + ) -> None: + if max_serialized_bytes <= 0: + raise ValueError("max_serialized_bytes must be positive") + if max_sanitized_bytes <= 0: + raise ValueError("max_sanitized_bytes must be positive") + if max_sanitized_segment_bytes <= 0: + raise ValueError("max_sanitized_segment_bytes must be positive") + if max_conservative_tokens <= 0: + raise ValueError("max_conservative_tokens must be positive") + if max_granted_serialized_bytes is not None and max_granted_serialized_bytes <= 0: + raise ValueError("max_granted_serialized_bytes must be positive") + if ( + max_granted_conservative_tokens is not None + and max_granted_conservative_tokens <= 0 + ): + raise ValueError("max_granted_conservative_tokens must be positive") + if conservative_chars_per_token <= 0: + raise ValueError("conservative_chars_per_token must be positive") + self._state_dir = Path(state_dir) + self._receipt_path = self._state_dir / "llm-egress-receipts.jsonl" + self._max_serialized_bytes = max_serialized_bytes + self._max_sanitized_bytes = max_sanitized_bytes + self._max_sanitized_segment_bytes = max_sanitized_segment_bytes + self._max_conservative_tokens = max_conservative_tokens + self._max_granted_serialized_bytes = ( + max_serialized_bytes + if max_granted_serialized_bytes is None + else max_granted_serialized_bytes + ) + self._max_granted_conservative_tokens = ( + max_conservative_tokens + if max_granted_conservative_tokens is None + else max_granted_conservative_tokens + ) + self._conservative_chars_per_token = conservative_chars_per_token + self._policy_digest = str(policy_digest or "") + self._exact_secret_values = tuple( + dict.fromkeys(value for value in exact_secret_values if isinstance(value, str) and value) + ) + self._static_literal_hashes_by_policy = { + str(policy_digest): frozenset( + digest + for digest in digests + if isinstance(digest, str) and re.fullmatch(r"[0-9a-f]{64}", digest) + ) + for policy_digest, digests in (static_literal_hashes_by_policy or {}).items() + } + + def preflight( + self, + request: Mapping[str, Any] | TypedOutboundRequest, + route: Any, + *, + grants: Sequence[SourceGrant] = (), + ) -> EgressDecision: + """Preserve the content-free Task 1 decision interface.""" + + return self.authorize(request, route, grants=grants).decision + + def authorize( + self, + request: Mapping[str, Any] | TypedOutboundRequest, + route: Any, + *, + grants: Sequence[SourceGrant] = (), + ) -> AuthorizedEgress: + """Construct and authorize immutable provider bytes or fail closed. + + Remote and unknown destinations accept only ``TypedOutboundRequest``. + Source text is loaded from verified grants while constructing the + logical request, so no independent raw source-bearing payload exists + for a caller to send after preflight. + """ + + provider = str(_route_value(route, "provider", "")) + model = str(_route_value(route, "model", "")) + destination = classify_destination( + provider, + _route_value(route, "base_url"), + _route_value(route, "api_mode"), + ) + base_url = str(_route_value(route, "base_url") or "") + api_mode = str(_route_value(route, "api_mode") or "") + typed_request = request if isinstance(request, TypedOutboundRequest) else None + if typed_request is not None: + session_id = typed_request.session_id + turn_id = typed_request.turn_id + request_id = typed_request.request_id + policy_digest = typed_request.policy_digest + else: + session_id = _request_identity(request, "session_id") + turn_id = _request_identity(request, "turn_id") + request_id = _request_identity(request, "request_id") + policy_digest = _request_identity(request, "policy_digest") + + reasons: list[str] = [] + valid_grants: list[SourceGrant] = [] + grant_contents: dict[str, tuple[SourceGrant, bytes]] = {} + grant_reasons: list[str] = [] + source_segment_count = 0 + sanitized_only = False + if typed_request is not None: + sanitized_shape, sanitized_count = _is_strict_sanitized_only_payload( + typed_request.payload + ) + sanitized_only = sanitized_shape and sanitized_count > 0 + + if destination == DestinationClass.UNKNOWN: + reasons.append("unknown_destination") + if destination in {DestinationClass.REMOTE, DestinationClass.UNKNOWN} and not all( + (session_id, turn_id, request_id, policy_digest) + ): + reasons.append("missing_request_identity") + if self._policy_digest and policy_digest != self._policy_digest: + reasons.append("policy_digest_mismatch") + if destination in {DestinationClass.REMOTE, DestinationClass.UNKNOWN}: + if typed_request is None: + reasons.append("typed_request_required") + # The sole grantless remote lane is a structurally verified request + # whose every text leaf is an explicit bounded SanitizedSegment. + # Passing even one purported grant opts back into exact source + # validation so malformed or unbound authority cannot be ignored. + if grants or not sanitized_only: + grant_reasons, valid_grants, grant_contents = self._validate_grants( + grants, + session_id=session_id, + turn_id=turn_id, + request_id=request_id, + policy_digest=policy_digest, + ) + reasons.extend(grant_reasons) + + if typed_request is not None: + ( + logical_request, + construction_reasons, + source_segment_count, + scan_values, + base64_scan_values, + ) = ( + self._construct_typed_request( + typed_request, + grant_contents, + allow_sanitized_segments=True, + ) + ) + reasons.extend(construction_reasons) + else: + logical_request = request + scan_values = request + base64_scan_values = request + + try: + serialized = json.dumps( + logical_request, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError, OverflowError): + decision = EgressDecision( + allowed=False, + destination_class=destination, + provider=provider, + model=model, + payload_sha256="", + serialized_bytes=0, + estimated_tokens=0, + source_grant_count=len(valid_grants), + source_segment_count=source_segment_count, + session_id=session_id, + turn_id=turn_id, + request_id=request_id, + policy_digest=policy_digest, + reason_codes=("serialization_failed",), + base_url=base_url, + api_mode=api_mode, + grant_digests=tuple(source_grant_digest(grant) for grant in valid_grants), + ) + self._block(decision, valid_grants) + + serialized_bytes = len(serialized) + estimated_tokens = ( + serialized_bytes + self._conservative_chars_per_token - 1 + ) // self._conservative_chars_per_token + use_granted_caps = bool( + typed_request is not None + and source_segment_count > 0 + and valid_grants + and not grant_reasons + ) + serialized_byte_cap = ( + self._max_granted_serialized_bytes + if use_granted_caps + else self._max_serialized_bytes + ) + conservative_token_cap = ( + self._max_granted_conservative_tokens + if use_granted_caps + else self._max_conservative_tokens + ) + if serialized_bytes > serialized_byte_cap: + reasons.append("serialized_bytes_exceeded") + if estimated_tokens > conservative_token_cap: + reasons.append("token_cap_exceeded") + + if destination in {DestinationClass.REMOTE, DestinationClass.UNKNOWN}: + try: + if _contains_secret(scan_values): + reasons.append("secret_detected") + except Exception: + reasons.append("redaction_failed") + try: + if _contains_exact_secret(scan_values, self._exact_secret_values): + reasons.append("exact_secret_detected") + except Exception: + reasons.append("exact_secret_scan_failed") + try: + if _contains_canonical_base64(base64_scan_values): + reasons.append("base64_payload") + except Exception: + reasons.append("base64_scan_failed") + try: + if _contains_private_absolute_path(scan_values): + reasons.append("private_absolute_path") + except Exception: + reasons.append("private_path_scan_failed") + + decision = EgressDecision( + allowed=not reasons, + destination_class=destination, + provider=provider, + model=model, + payload_sha256=sha256(serialized).hexdigest(), + serialized_bytes=serialized_bytes, + estimated_tokens=estimated_tokens, + source_grant_count=len(valid_grants), + source_segment_count=source_segment_count, + session_id=session_id, + turn_id=turn_id, + request_id=request_id, + policy_digest=policy_digest, + reason_codes=tuple(dict.fromkeys(reasons)), + base_url=base_url, + api_mode=api_mode, + grant_digests=tuple(source_grant_digest(grant) for grant in valid_grants), + ) + if not decision.allowed: + self._block(decision, valid_grants) + + try: + self._append_receipt(decision, valid_grants) + except OSError: + raise EgressBlocked( + replace(decision, allowed=False, reason_codes=("receipt_unavailable",)) + ) from None + return AuthorizedEgress(decision=decision, payload_bytes=serialized) + + def _validate_grants( + self, + grants: Sequence[SourceGrant], + *, + session_id: str, + turn_id: str, + request_id: str, + policy_digest: str, + ) -> tuple[list[str], list[SourceGrant], dict[str, tuple[SourceGrant, bytes]]]: + reasons: list[str] = [] + valid: list[SourceGrant] = [] + contents: dict[str, tuple[SourceGrant, bytes]] = {} + if not grants: + return ["untrusted_provenance"], valid, contents + + for candidate in grants: + if not isinstance(candidate, SourceGrant): + reasons.append("untrusted_provenance") + continue + grant = candidate + if ( + grant.session_id != session_id + or grant.turn_id != turn_id + or grant.request_id != request_id + or grant.policy_digest != policy_digest + ): + reasons.append("grant_binding_mismatch") + continue + if ( + grant.line_start < 1 + or grant.line_end < grant.line_start + or grant.byte_count < 0 + or not re.fullmatch(r"[0-9a-f]{64}", grant.content_sha256) + ): + reasons.append("invalid_source_grant") + continue + display = Path(grant.display_path) + if display.is_absolute() or ".." in display.parts: + reasons.append("invalid_display_path") + continue + try: + canonical = Path(grant.canonical_path) + resolved = canonical.resolve(strict=True) + except (OSError, RuntimeError, ValueError, TypeError): + reasons.append("source_unavailable") + continue + if not canonical.is_absolute() or canonical != resolved: + reasons.append("source_path_not_canonical") + continue + try: + blocked = get_read_block_error(str(resolved)) + except Exception: + reasons.append("source_policy_unavailable") + continue + if blocked is not None: + reasons.append("sensitive_path") + continue + try: + lines = resolved.read_bytes().splitlines(keepends=True) + except OSError: + reasons.append("source_unavailable") + continue + if grant.line_end > len(lines): + reasons.append("source_range_mismatch") + continue + content = b"".join(lines[grant.line_start - 1 : grant.line_end]) + if len(content) != grant.byte_count or sha256(content).hexdigest() != grant.content_sha256: + reasons.append("source_hash_mismatch") + continue + valid.append(grant) + contents[source_grant_digest(grant)] = (grant, content) + + if not valid and "untrusted_provenance" not in reasons: + reasons.append("untrusted_provenance") + return reasons, valid, contents + + def _construct_typed_request( + self, + request: TypedOutboundRequest, + grant_contents: Mapping[str, tuple[SourceGrant, bytes]], + *, + allow_sanitized_segments: bool = False, + ) -> tuple[Mapping[str, Any], list[str], int, list[str], list[str]]: + """Build a plain JSON request exclusively from typed segment nodes.""" + + reasons: list[str] = [] + referenced_grants: set[str] = set() + source_segment_count = 0 + sanitized_bytes = 0 + scan_values: list[str] = [] + base64_scan_values: list[str] = [] + allowed_static_hashes = self._static_literal_hashes_by_policy.get( + request.policy_digest, + frozenset(), + ) + + def require_static_literal( + text: str, + *, + scan_base64: bool = True, + scan_secret: bool = True, + ) -> None: + # Provenance authorization and content safety are independent. + # Every rendered text atom is scanned again immediately before + # authorization, including exact policy-bound static literals and + # structural keys/scalars. + if scan_secret: + scan_values.append(text) + if scan_base64: + base64_scan_values.append(text) + if static_literal_sha256(text) not in allowed_static_hashes: + reasons.append("static_literal_not_allowed") + + def render_text_segment( + segment: ( + LiteralSegment + | SanitizedSegment + | GeneratedContextSegment + | CodexReasoningReplaySegment + | ValidatedToolSyntaxSegment + | SourceBoundSegment + | SourcePresentationSegment + | UntrustedProvenanceSegment + ), + ) -> str: + nonlocal sanitized_bytes, source_segment_count + if isinstance(segment, LiteralSegment): + if not isinstance(segment.text, str): + reasons.append("invalid_literal_segment") + return "" + require_static_literal(segment.text) + encoded_literal = segment.text.encode("utf-8") + if any(content and content in encoded_literal for _, content in grant_contents.values()): + reasons.append("source_bytes_in_literal") + return segment.text + if isinstance(segment, SanitizedSegment): + if not allow_sanitized_segments: + reasons.append("sanitized_segment_forbidden") + if isinstance(segment.text, str): + encoded = segment.text.encode("utf-8") + if len(encoded) > self._max_sanitized_segment_bytes: + reasons.append("sanitized_segment_bytes_exceeded") + sanitized_bytes += len(encoded) + if sanitized_bytes > self._max_sanitized_bytes: + reasons.append("sanitized_bytes_exceeded") + if any( + _contains_grant_substring(content, encoded) + for _, content in grant_contents.values() + ): + reasons.append("source_bytes_in_sanitized_segment") + scan_values.append(segment.text) + base64_scan_values.append(segment.text) + return segment.text + reasons.append("invalid_literal_segment") + return "" + if isinstance(segment, GeneratedContextSegment): + if not isinstance(segment.text, str): + reasons.append("invalid_generated_context_segment") + return "" + # The constructor redacts path/base64-shaped text. Keep the + # final scans, especially secret detection, as defense in + # depth, but do not charge generated context to the smaller + # untrusted-text budget. + scan_values.append(segment.text) + base64_scan_values.append( + _generated_context_text_for_base64_scan(segment.text) + ) + return segment.text + if isinstance(segment, CodexReasoningReplaySegment): + if not _CODEX_ENCRYPTED_REASONING_REPLAY.fullmatch(segment.text): + reasons.append("invalid_codex_reasoning_replay") + return "" + # This opaque token is produced by the Codex Responses API and + # is typed only for a reasoning item on that route. It must be + # replayed verbatim for cache and reasoning continuity, but it + # is not an independently usable credential payload. + return segment.text + if isinstance(segment, ValidatedToolSyntaxSegment): + try: + text = validate_tool_syntax(segment.text, segment.syntax_kind) + except (TypeError, ValueError): + reasons.append("invalid_tool_syntax_segment") + return "" + encoded = text.encode("utf-8") + if len(encoded) > self._max_sanitized_segment_bytes: + reasons.append("sanitized_segment_bytes_exceeded") + sanitized_bytes += len(encoded) + if sanitized_bytes > self._max_sanitized_bytes: + reasons.append("sanitized_bytes_exceeded") + scan_values.append(text) + return text + if isinstance(segment, SourceBoundSegment): + grant_and_content = grant_contents.get(segment.source_grant_digest) + if grant_and_content is None: + reasons.append("source_segment_grant_mismatch") + return "" + try: + text = grant_and_content[1].decode("utf-8") + except UnicodeDecodeError: + reasons.append("source_segment_not_text") + return "" + referenced_grants.add(segment.source_grant_digest) + source_segment_count += 1 + scan_values.append(_source_text_for_secret_scan(text)) + base64_scan_values.append(_source_text_for_base64_scan(text)) + return text + if isinstance(segment, SourcePresentationSegment): + grant_and_content = grant_contents.get(segment.source_grant_digest) + if grant_and_content is None: + reasons.append("source_segment_grant_mismatch") + return "" + if segment.presentation_kind != "read_file_json_v1": + reasons.append("invalid_source_presentation") + return "" + try: + raw_text = grant_and_content[1].decode("utf-8") + expected_content = "\n".join( + f"{line_number}|{line}" + for line_number, line in enumerate( + raw_text.split("\n"), + start=grant_and_content[0].line_start, + ) + ) + parsed = json.loads(segment.text) + except (UnicodeDecodeError, TypeError, ValueError, json.JSONDecodeError): + reasons.append("invalid_source_presentation") + return "" + if not isinstance(parsed, dict) or parsed.get("content") != expected_content: + reasons.append("invalid_source_presentation") + return "" + referenced_grants.add(segment.source_grant_digest) + source_segment_count += 1 + scan_values.append(_source_text_for_secret_scan(raw_text)) + base64_scan_values.append( + _source_text_for_base64_scan( + raw_text, allow_fixed_source_literals=True + ) + ) + return segment.text + if isinstance(segment, UntrustedProvenanceSegment): + reasons.append("untrusted_provenance") + return "" + reasons.append("invalid_source_segment") + return "" + + def render(value: Any) -> Any: + if isinstance( + value, + ( + LiteralSegment, + SanitizedSegment, + GeneratedContextSegment, + CodexReasoningReplaySegment, + ValidatedToolSyntaxSegment, + SourceBoundSegment, + SourcePresentationSegment, + UntrustedProvenanceSegment, + ), + ): + return render_text_segment(value) + if isinstance(value, OutboundText): + rendered_parts: list[str] = [] + adjacent_sanitized: list[str] = [] + adjacent_non_source: list[str] = [] + + def flush_adjacent_sanitized() -> None: + if len(adjacent_sanitized) > 1: + combined = "".join(adjacent_sanitized) + # Segment caps are transport bounds, not scan + # boundaries. Re-scan each reconstructed contiguous + # sanitized span so splitting cannot conceal a secret, + # private path, or encoding across adjacent pieces. + scan_values.append(combined) + base64_scan_values.append(combined) + adjacent_sanitized.clear() + + def flush_adjacent_non_source() -> None: + if len(adjacent_non_source) > 1: + # Structural typing may exclude an exact validated + # atom from Base64 classification, but it must never + # split a secret or private path across scan values. + scan_values.append("".join(adjacent_non_source)) + adjacent_non_source.clear() + + for segment in value.segments: + rendered = render_text_segment(segment) + rendered_parts.append(rendered) + if isinstance(segment, SanitizedSegment): + adjacent_sanitized.append(rendered) + else: + flush_adjacent_sanitized() + if isinstance( + segment, + ( + LiteralSegment, + SanitizedSegment, + GeneratedContextSegment, + CodexReasoningReplaySegment, + ValidatedToolSyntaxSegment, + ), + ): + adjacent_non_source.append(rendered) + else: + flush_adjacent_non_source() + flush_adjacent_sanitized() + flush_adjacent_non_source() + return "".join(rendered_parts) + if isinstance(value, Mapping): + rendered: dict[str, Any] = {} + for key, item in value.items(): + if isinstance(key, GeneratedContextKey): + rendered_key = key.text + if not isinstance(rendered_key, str): + reasons.append("invalid_generated_context_key") + continue + require_static_literal(rendered_key, scan_base64=False) + elif isinstance(key, str): + rendered_key = key + # Mapping keys are policy-bound request structure, not + # caller-supplied payload. Scanning them as Base64 + # turns legitimate protocol fields such as + # ``response_item_id`` into false-positive payloads. + require_static_literal(rendered_key, scan_base64=False) + else: + reasons.append("invalid_request_key") + continue + rendered[rendered_key] = render(item) + return rendered + if isinstance(value, (list, tuple)): + return [render(item) for item in value] + if isinstance(value, float) and not math.isfinite(value): + reasons.append("non_finite_number") + return None + if value is None or isinstance(value, (bool, int, float)): + # JSON scalar controls (for example ``max_tokens=4096``) are + # rendered as unquoted JSON values, never caller-supplied + # text. Scanning their string representation as a standalone + # base64 candidate turns ordinary numeric limits into false + # egress blocks ("4096" is a valid four-character base64 + # alphabet member). Keep them policy-bound and in the secret + # scan, but do not apply a text-payload base64 heuristic. + require_static_literal( + json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")), + scan_base64=False, + ) + return value + # In particular, raw strings and bytes are not remote request + # material. Every outbound string must have a typed owner. + reasons.append("untyped_request_value") + return None + + rendered_payload = render(request.payload) + if not isinstance(rendered_payload, Mapping): + reasons.append("invalid_typed_request_root") + rendered_payload = {} + else: + bound_identities = { + "session_id": request.session_id, + "turn_id": request.turn_id, + "request_id": request.request_id, + "policy_digest": request.policy_digest, + } + if any( + field in rendered_payload and rendered_payload[field] != expected + for field, expected in bound_identities.items() + ): + reasons.append("request_identity_mismatch") + if set(grant_contents) - referenced_grants: + reasons.append("source_grant_unbound") + return ( + rendered_payload, + reasons, + source_segment_count, + scan_values, + base64_scan_values, + ) + + def _block( + self, + decision: EgressDecision, + grants: Sequence[SourceGrant] = (), + ) -> None: + try: + self._append_receipt(decision, grants) + except OSError: + decision = replace( + decision, + reason_codes=tuple(dict.fromkeys((*decision.reason_codes, "receipt_unavailable"))), + ) + raise EgressBlocked(decision) + + def _append_receipt( + self, + decision: EgressDecision, + grants: Sequence[SourceGrant], + ) -> None: + try: + if self._state_dir.is_symlink(): + raise OSError("egress state directory must not be a symlink") + except OSError: + raise + self._state_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + receipt = asdict(decision) + receipt["destination_class"] = decision.destination_class.value + receipt["decision"] = "allow" if decision.allowed else "block" + for field in ( + "provider", + "model", + "base_url", + "api_mode", + "session_id", + "turn_id", + "request_id", + "policy_digest", + ): + receipt[field] = _receipt_identifier(receipt[field]) + receipt["source_grants"] = [ + { + "line_start": grant.line_start, + "line_end": grant.line_end, + "content_sha256": grant.content_sha256, + "byte_count": grant.byte_count, + } + for grant in grants + ] + flags = os.O_APPEND | os.O_CREAT | os.O_RDWR + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + with exclusive_file_lock(self._receipt_path.with_suffix(".lock")): + fd = os.open(self._receipt_path, flags, 0o600) + try: + secure_file_descriptor_permissions(fd) + file_size = os.fstat(fd).st_size + previous_hash = "" + if file_size: + read_start = max(0, file_size - 131_072) + os.lseek(fd, read_start, os.SEEK_SET) + prior_chunk = os.read(fd, file_size - read_start) + prior_lines = prior_chunk.splitlines() + if prior_lines: + previous_hash = sha256(prior_lines[-1]).hexdigest() + receipt["receipt_prev_sha256"] = previous_hash + receipt_material = json.dumps( + receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True + ).encode("utf-8") + receipt["receipt_sha256"] = sha256( + previous_hash.encode("ascii") + receipt_material + ).hexdigest() + encoded = ( + json.dumps(receipt, ensure_ascii=True, separators=(",", ":"), sort_keys=True) + + "\n" + ).encode("utf-8") + os.lseek(fd, 0, os.SEEK_END) + if os.write(fd, encoded) != len(encoded): + raise OSError("short receipt write") + finally: + os.close(fd) + + +__all__ = [ + "AuthorizedEgress", + "DestinationClass", + "EgressBlocked", + "EgressDecision", + "GeneratedContextKey", + "GeneratedContextSegment", + "LLMEgressFirewall", + "LiteralSegment", + "OutboundText", + "SanitizedSegment", + "SourceBoundSegment", + "SourcePresentationSegment", + "SourceGrant", + "TypedOutboundRequest", + "UntrustedProvenanceSegment", + "ValidatedToolSyntaxSegment", + "classify_destination", + "redact_remote_unsafe_text", + "source_grant_digest", + "static_literal_sha256", + "validate_tool_syntax", +] diff --git a/agent/llm_egress_runtime.py b/agent/llm_egress_runtime.py new file mode 100644 index 0000000000000..c5b4932a91e85 --- /dev/null +++ b/agent/llm_egress_runtime.py @@ -0,0 +1,4484 @@ +"""Final provider-boundary enforcement for source-bound LLM egress.""" + +from __future__ import annotations + +import json +import logging +import math +import os +import re +import shlex +from hashlib import sha256 +from pathlib import Path, PurePosixPath +from types import MappingProxyType +from types import SimpleNamespace +from typing import Any, Callable, Mapping, Sequence +from urllib.parse import parse_qs, urlsplit + +from agent.llm_egress_firewall import ( + AuthorizedEgress, + CodexReasoningReplaySegment, + EgressBlocked, + LLMEgressFirewall, + LiteralSegment, + OutboundText, + SanitizedSegment, + SourceBoundSegment, + SourcePresentationSegment, + SourceGrant, + TypedOutboundRequest, + UntrustedProvenanceSegment, + ValidatedToolSyntaxSegment, + DestinationClass, + GeneratedContextKey, + GeneratedContextSegment, + classify_destination, + source_grant_digest, + static_literal_sha256, + validate_sanitized_text, + content_free_violation_locations, + redact_remote_unsafe_text, + validate_tool_syntax, +) +from agent.message_sanitization import tool_result_id_variants +from agent.redact import redact_sensitive_text +from agent.source_provenance import DEFAULT_POLICY_DIGEST, SourceProvenanceRegistry + + +# Timeout is a non-content SDK control. Header/query values remain in the +# authorized JSON body so credentials or other caller-controlled text cannot +# be appended after the firewall receipt is written. +_SDK_CONTROL_KEYS = frozenset({"timeout"}) +_INTERNAL_EGRESS_KEYS = frozenset({"_hermes_source_provenance"}) +_PROTOCOL_LITERAL_FIELDS = frozenset({"role", "type"}) +_TOOL_PROTOCOL_IDENTIFIER_FIELDS = frozenset( + {"id", "call_id", "tool_call_id", "response_item_id"} +) +_PROTOCOL_LITERAL_VALUES = frozenset({ + "assistant", + "computer_call_output", + "developer", + "function_call", + "function_call_output", + "input_image", + "input_text", + "output_text", + "reasoning", + "system", + "tool", + "user", +}) +_PROTECTED_REMOTE_PROVIDERS = frozenset({ + "anthropic", + "openai-codex", + "nous", + "nous-portal", + "nousresearch", +}) +logger = logging.getLogger(__name__) + +_VALIDATED_SYNTAX_TOOL_NAMES = frozenset({"terminal"}) +_REMOTE_KANBAN_PROJECTION_TOOL_NAMES = frozenset({"kanban_show"}) +_REMOTE_KANBAN_ATTACHMENT_TOOL_NAMES = frozenset({"kanban_attachments"}) +# Local action results are safe to replay only as bounded outcomes, and only +# when the result is bound to the exact preceding call. Browser Use runs +# through ``browser_exec`` rather than ``terminal``; omitting it here makes a +# protected worker treat its own browser result as untrusted provenance and +# fail on otherwise harmless page identifiers or encoded-looking text. +_REMOTE_KANBAN_TERMINAL_REPLAY_TOOL_NAMES = frozenset({"terminal", "browser_exec"}) +_REMOTE_KANBAN_SEARCH_PROJECTION_TOOL_NAMES = frozenset({"search_files"}) +# Both catalog bridge calls return model-readable tool schemas/descriptions. +# Protected workers must replay only the bounded local outcome; otherwise a +# tool_describe result is treated as untrusted provider content and can trip +# the egress firewall on harmless schema words. +_REMOTE_KANBAN_TOOL_SEARCH_PROJECTION_TOOL_NAMES = frozenset( + {"tool_search", "tool_describe"} +) +_REMOTE_KANBAN_READ_FILE_PROJECTION_TOOL_NAMES = frozenset({"read_file"}) +_REMOTE_KANBAN_WEB_REPLAY_TOOL_NAMES = frozenset({"web_extract", "web_search"}) +_REMOTE_KANBAN_FILE_MUTATION_REPLAY_TOOL_NAMES = frozenset({"patch", "write_file"}) +_REMOTE_KANBAN_LIFECYCLE_TOOL_NAMES = frozenset( + { + "kanban_attach", + "kanban_attach_url", + "kanban_block", + "kanban_comment", + "kanban_complete", + "kanban_heartbeat", + "kanban_link", + "kanban_request_changes", + "kanban_request_review", + } +) +_REMOTE_KANBAN_READONLY_REPLAY_TOOL_NAMES = frozenset( + { + "kanban_show", + "kanban_attachments", + "search_files", + "read_file", + "web_extract", + "web_search", + } +) +_GITHUB_PR_FEEDBACK_TERMINAL_SUBCOMMANDS = frozenset( + { + "inspect-pr", + "complete-feedback", + "retire-feedback", + "submit-review", + "status", + } +) +_GITHUB_PR_FEEDBACK_TERMINAL_RESULT_KEYS = frozenset( + { + "base_branch", + "base_sha", + "codex_retrigger_status", + "event", + "expected_head_sha", + "fallback", + "feedback_body_excerpt", + "feedback_id", + "feedback_is_bot", + "feedback_kind", + "feedback_reviewer", + "head_ref_name", + "head_repository", + "head_sha", + "local_ci_status", + "number", + "observed_head_sha", + "pr_number", + "pr_state", + "state", + "task_id", + "reason", + "repository", + "resolved_head_sha", + "review_thread_resolved", + "status", + "error_excerpt", + "receipt_id", + "manifest_digest", + "handoff_reason", + "handoff_status", + "repair_status", + "retryable", + "command_count", + } +) +_GITHUB_LIST_TERMINAL_MAX_ROWS = 100 +_GITHUB_LIST_TERMINAL_MAX_ITEM_BYTES = 512 +_GITHUB_LIST_TERMINAL_MAX_OUTPUT_BYTES = 10_240 +_GIT_GREP_TERMINAL_MAX_MATCHES = 200 +_GIT_DIFF_NAME_ONLY_MAX_FILES = 200 +_GIT_REVIEW_SUMMARY_MAX_FILES = 200 +_PYTEST_DIAGNOSTIC_MAX_LINES = 32 +_PYTEST_DIAGNOSTIC_MAX_BYTES = 4096 +_FILE_MUTATION_ERROR_MAX_BYTES = 1024 +_GITHUB_API_EXTRACT_ARGUMENT_REPLAY = ( + '{"urls":["https://api.github.com/repos///"]}' +) +_GITHUB_API_PAGINATE_ARGUMENT_REPLAY = ( + '{"command":"gh api --paginate GitHub REST list (details omitted)"}' +) +_GITHUB_API_CURL_ARGUMENT_REPLAY = ( + '{"command":"curl GitHub REST list (details omitted)"}' +) +_GITHUB_PLAIN_LIST_OUTPUT_REPLAY = ( + "GitHub list output omitted; use --json for bounded fields." +) +_REJECTED_TERMINAL_COMMAND_REPLAY = json.dumps( + {"command": ""}, separators=(",", ":") +) +_GIT_WORKSPACE_DIAGNOSTIC_REPLAY = ( + "git workspace diagnostic completed locally; raw paths and commit subjects " + "were omitted from remote replay." +) +_READ_FILE_REPLAY_ELISION = ( + "read_file completed locally, but its raw content cannot be replayed on " + "this protected route. Request only the needed narrow range again." +) +_STRUCTURED_SEARCH_REPLAY_ELISION = ( + "search completed locally; structured output omitted from remote replay." +) +_FILE_MUTATION_REPLAY_ELISION = ( + "local file mutation completed; raw source and diff omitted from remote replay. " + "Inspect git diff and status for the exact result." +) +_FILE_MUTATION_ARGUMENT_REPLAY = json.dumps( + {"path": "", "content": "omitted from remote replay"}, + separators=(",", ":"), +) +_REMOTE_KANBAN_SECRET_ASSIGNMENT = re.compile( + r"(?i)\b(token|secret|password|api[_-]?key)\s*[:=]\s*[^\s,}\"']+" +) +_REMOTE_KANBAN_PROJECTION_ELISION = ( + "kanban_show completed locally. The bounded task assignment is already " + "present in your worker context; do not request or repeat the raw board " + "record remotely. Continue with the assigned work or use a lifecycle tool." +) +_REMOTE_KANBAN_TASK_SPEC_VERSION = "v1" +_REMOTE_KANBAN_TASK_TITLE_MAX_BYTES = 1024 +_REMOTE_KANBAN_TASK_BODY_MAX_BYTES = 8 * 1024 +_REMOTE_KANBAN_ATTACHMENT_ELISION = ( + "kanban_attachments completed locally; attachment metadata and contents " + "were omitted from remote replay. Continue with the assigned work or use a lifecycle tool." +) +_REMOTE_KANBAN_LIFECYCLE_ELISION = ( + "Kanban lifecycle action completed locally; its raw control-plane result " + "was omitted from remote replay." +) + + +def _project_bound_kanban_lifecycle(value: str) -> GeneratedContextSegment: + """Replay only a fixed outcome for an exact local lifecycle call.""" + + # Lifecycle results can include comment text, paths, opaque ids, or + # backend errors. The worker only needs the fact that its local action + # returned; exact call-id binding is enforced by the caller. + return GeneratedContextSegment(_REMOTE_KANBAN_LIFECYCLE_ELISION) + + +def _project_bound_kanban_show(value: str) -> GeneratedContextSegment: + """Expose only the redacted current assignment needed by a remote worker.""" + + try: + payload = json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError): + return GeneratedContextSegment(_REMOTE_KANBAN_PROJECTION_ELISION) + task = payload.get("task") if isinstance(payload, dict) else None + if not isinstance(task, dict): + return GeneratedContextSegment(_REMOTE_KANBAN_PROJECTION_ELISION) + + # Only the exact versioned producer contract may carry assignment text. + # Forged/unbound board-shaped JSON stays on the elision path. The producer + # has already capped the fields and the redaction/final scans remain + # mandatory before this generated context can leave the host. + task_spec = payload.get("protected_task_spec") + + def bounded_text(item: Any, max_bytes: int) -> str: + text = item if isinstance(item, str) else "" + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + suffix = "\n" + budget = max(0, max_bytes - len(suffix.encode("utf-8"))) + return encoded[:budget].decode("utf-8", errors="ignore") + suffix + + projected_task: dict[str, Any] = { + key: task[key] + for key in ("status", "workspace_access") + if key in task + } + if ( + isinstance(task_spec, dict) + and task_spec.get("version") == _REMOTE_KANBAN_TASK_SPEC_VERSION + ): + projected_task.update( + { + "title": bounded_text( + task_spec.get("title"), _REMOTE_KANBAN_TASK_TITLE_MAX_BYTES + ), + "body": bounded_text( + task_spec.get("body"), _REMOTE_KANBAN_TASK_BODY_MAX_BYTES + ), + } + ) + + projection = { + "task": projected_task, + "worker_instruction": ( + "Use the dispatcher-assigned current workspace. Do not invent or search " + "for alternate worktrees; report an unresolved assignment and stop." + ), + } + safe = redact_remote_unsafe_text( + redact_sensitive_text(json.dumps(projection, sort_keys=True), force=True) + ) + safe = _REMOTE_KANBAN_SECRET_ASSIGNMENT.sub(r"\1=", safe) + return GeneratedContextSegment( + "kanban_show completed locally. Bounded sanitized task projection:\n" + safe + ) + + +def _project_bound_kanban_attachments(value: str) -> GeneratedContextSegment: + """Elide attachment payloads while preserving exact call/result binding.""" + + # Attachment records can contain source excerpts, credentials, and opaque + # blobs. The worker already has the bounded task assignment; replaying + # attachment content is unnecessary and would make the protected route + # pay for a retry when provenance cannot be established. + return GeneratedContextSegment(_REMOTE_KANBAN_ATTACHMENT_ELISION) + + +def _project_bound_search_files(value: str) -> GeneratedContextSegment: + """Retain search locations without replaying matched source bytes. + + ``search_files`` necessarily returns excerpts of local source. A protected + worker may use the count and (when compact) the file/line locations to + choose a narrow ``read_file`` request, whose exact bytes are independently + source-provenance bound. Never parse or replay ``matches_text``: it is a + dense display format containing source content. + """ + + try: + payload = json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError): + payload = None + if not isinstance(payload, Mapping): + return GeneratedContextSegment( + "search_files completed locally. Its raw result was omitted from the " + "remote replay; narrow the search or use read_file for a known path." + ) + + projection: dict[str, Any] = {"search_files_projection": "locations-v1"} + total_count = payload.get("total_count") + if isinstance(total_count, int) and not isinstance(total_count, bool): + projection["total_count"] = max(0, min(total_count, 1_000_000)) + if payload.get("truncated") is True: + projection["truncated"] = True + + raw_files = payload.get("files") + if isinstance(raw_files, list): + files: list[str] = [] + for raw_path in raw_files[:100]: + if not isinstance(raw_path, str) or not raw_path or len(raw_path) > 512: + continue + normalized = raw_path[2:] if raw_path.startswith("./") else raw_path + path = PurePosixPath(normalized) + if ( + path.is_absolute() + or "\\" in normalized + or any( + part in {"", ".", ".."} + or (part.startswith(".") and part != ".github") + for part in path.parts + ) + ): + continue + safe_path = redact_remote_unsafe_text( + redact_sensitive_text(path.as_posix(), force=True) + ) + if safe_path == path.as_posix(): + files.append(safe_path) + if files: + projection["files"] = files + + raw_matches = payload.get("matches") + if isinstance(raw_matches, list): + matches: list[dict[str, Any]] = [] + for raw_match in raw_matches[:100]: + if not isinstance(raw_match, Mapping): + continue + path = raw_match.get("path") + line = raw_match.get("line") + if not isinstance(path, str) or not isinstance(line, int) or isinstance(line, bool): + continue + safe_path = redact_remote_unsafe_text( + redact_sensitive_text(path, force=True) + ) + matches.append({"path": safe_path, "line": max(1, min(line, 10_000_000))}) + if matches: + projection["matches"] = matches + + safe = redact_remote_unsafe_text( + redact_sensitive_text( + json.dumps(projection, ensure_ascii=False, separators=(",", ":")), + force=True, + ) + ) + return GeneratedContextSegment(safe) + + +def _project_bound_tool_search(value: str) -> GeneratedContextSegment: + """Replay only the bounded outcome of local tool catalog discovery.""" + + return GeneratedContextSegment( + "tool_search completed locally. Its catalog result was omitted from " + "remote replay; use the already connected terminal tool." + ) + + +def _project_web_search_replay(value: str) -> SanitizedSegment: + """Keep bounded public result identity, never raw page/search excerpts.""" + + start = value.find("{") if isinstance(value, str) else -1 + end = value.rfind("}") if isinstance(value, str) else -1 + try: + payload = json.loads(value[start : end + 1]) if 0 <= start <= end else None + except (TypeError, ValueError, json.JSONDecodeError): + payload = None + + raw_results: Any = None + if isinstance(payload, Mapping): + search = payload.get("search") + if isinstance(search, Mapping): + raw_results = search.get("web") + if raw_results is None: + raw_results = payload.get("results") + + results: list[dict[str, str]] = [] + if isinstance(raw_results, list): + for raw in raw_results[:20]: + if not isinstance(raw, Mapping): + continue + projected: dict[str, str] = {} + for key in ("url", "title"): + item = raw.get(key) + if not isinstance(item, str): + continue + candidate = redact_remote_unsafe_text( + redact_sensitive_text( + item, + force=True, + redact_url_credentials=True, + ) + ) + try: + projected[key] = validate_sanitized_text(candidate, max_bytes=2_048) + except (TypeError, ValueError): + continue + if projected: + results.append(projected) + + projection = { + "kind": "web results", + "results": results, + "raw excerpts omitted": True, + } + rendered = json.dumps(projection, ensure_ascii=False, separators=(",", ":")) + return SanitizedSegment(validate_sanitized_text(rendered)) + + +_APPLICATION_IDENTIFIER_TOKEN = re.compile( + r"(? Any: + """Remove host paths from protected Kanban tool results before typing. + + This deliberately does not rewrite secrets or arbitrary encoded content; + those remain visible to the fail-closed firewall scans and are denied. + """ + + if isinstance(value, str): + text = value + for name in ( + "HERMES_KANBAN_CLAIM_LOCK", + "HERMES_KANBAN_RUN_ID", + "HERMES_SESSION_ID", + "HERMES_STREAM_STALE_GIVEUP", + "HERMES_TURN_LEASE_TIMEOUT", + ): + raw = os.environ.get(name) + if raw: + text = re.sub( + rf"(?m)^(?P