diff --git a/.env.example b/.env.example index 768eca50091..27cd9b3d9ca 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,9 @@ # Default workspace directory shown on first launch # HERMES_WEBUI_DEFAULT_WORKSPACE=~/workspace +# Optional model override. Leave unset to use the active Hermes provider default. +# HERMES_WEBUI_DEFAULT_MODEL= + # Base directory for all Hermes state (affects all paths above if set) # HERMES_HOME=~/.hermes diff --git a/.gitignore b/.gitignore index 28316280003..e0f68fbc71d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ archive/ !.env.docker.example .claude/ CLAUDE.md -AGENTS.md +AGENTS.local.md .cursorrules .windsurfrules .aider* diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..a916be0466e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,68 @@ +# Agent instructions for Hermes WebUI + +This file is the shared entry point for AI assistants working in this +repository. Keep it project-specific and safe to publish. Do not put personal +machine setup, private network details, credentials, tokens, or local-only +workflow notes here. + +## Read first + +Before making changes, read: + +1. `README.md` +2. `CONTRIBUTING.md` +3. `CHANGELOG.md` + +For architecture, testing, or setup work, also read the matching reference: + +- `ARCHITECTURE.md` for design constraints and current module layout +- `TESTING.md` for local verification commands and manual test guidance +- `docs/onboarding.md` for first-run onboarding behavior +- `docs/troubleshooting.md` for diagnostic flows + +## Onboarding and reinstall support + +If the task involves install, reinstall, bootstrap, first-run onboarding, +provider setup, local model server setup, Docker onboarding, WSL onboarding, or +support for a failed first run, read `docs/onboarding-agent-checklist.md` +before running commands or inspecting logs. + +Follow that checklist's safety rules: + +- use isolated `HERMES_HOME` and `HERMES_WEBUI_STATE_DIR` for trials unless the + human explicitly asks to use real state +- do not delete or overwrite a real `~/.hermes` directory without explicit + approval +- do not print API keys, OAuth tokens, cookies, full `.env` files, full + `auth.json` files, or password hashes +- collect non-secret status and log evidence before recommending a fix + +## Contribution style + +- Keep changes focused on one logical problem. +- Prefer the existing Python + vanilla JavaScript structure over new + dependencies or build steps. +- Update docs when changing setup, onboarding, runtime behavior, architecture, + or testing guidance. +- Update `CHANGELOG.md` for user-visible behavior, setup, workflow, or + documentation changes that should be release-note ready. +- For UI or UX changes, follow `CONTRIBUTING.md`: include before/after evidence + and test relevant responsive states. + +## Local state and secrets + +Hermes WebUI can read and write real agent state, sessions, workspaces, +credentials, and cron data. Treat local validation as potentially destructive +unless you have confirmed the active state directories. + +Prefer isolated trial state for experiments: + +```bash +HERMES_HOME=/tmp/hermes-webui-agent-home \ +HERMES_WEBUI_STATE_DIR=/tmp/hermes-webui-agent-state \ +HERMES_WEBUI_PORT=8789 \ +python3 bootstrap.py +``` + +Do not include private machine instructions in this tracked file. Use a +git-ignored local note for personal workflow details. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1d8f5a718da..a0f92b9f914 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -7,10 +7,10 @@ > > Keep this document updated as architecture changes are made. -> Current shipped build: `v0.50.245` (April 30, 2026). -> Automated coverage: 3309 tests via `pytest tests/ --collect-only -q`. CI runs on Python 3.11, 3.12, and 3.13 against every PR. +> Current shipped build: `v0.51.54` (May 13, 2026). +> Automated coverage: 5303 tests via `pytest tests/ --collect-only -q`. CI runs on Python 3.11, 3.12, and 3.13 against every PR. > -> Notable architecture state as of v0.50.245: workspace panel closed/open state is preloaded via a `documentElement` dataset marker before `style.css` paints to avoid first-load flash; transcript disclosure cards animate via transitionable `max-height`/`opacity` states; thinking cards share rounded bordered card chrome with tool cards (gold palette); incremental streaming-markdown via vendored `streaming-markdown@0.2.15` (no CDN); HTTP byte-range streaming for large media; SSE-driven session sidebar with `pending_user_message` + `active_stream_id` lifecycle tracking; configurable model badges (`primary` / `fallback N`) computed in `_build_configured_model_badges()` and provider-aware in the dropdown picker. +> Notable architecture state as of v0.51.54: the bootstrap and first-run onboarding flow own setup discovery; the default WebUI state directory is `~/.hermes/webui`; `ctl.sh` provides a daemon wrapper for homelab installs; chat streaming is still WebUI-owned SSE with stream-ownership guards, cancellation, async manual compression, and turn-journal audit plumbing; provider/model discovery is profile-aware with live-model cache invalidation and custom-provider scoping. --- @@ -43,42 +43,42 @@ actions. The topbar remains focused on conversation context and the workspace/fi ## 2. File Inventory / - server.py Thin routing shell + HTTP Handler + auth middleware. ~81 lines. + server.py Thin routing shell + HTTP Handler + auth middleware. ~446 lines. Delegates all route handling to api/routes.py. bootstrap.py One-shot launcher: optional agent install, deps, health wait, browser open. start.sh Thin wrapper around bootstrap.py for shell-based startup. - Dockerfile python:3.12-slim container image (~23 lines) - docker-compose.yml Compose config with named volume and optional auth (~22 lines) + Dockerfile python:3.12-slim container image (~89 lines) + docker-compose.yml Compose config with named volume and optional auth (~57 lines) .dockerignore Excludes .git, tests/, .env* from Docker builds api/ __init__.py Package marker - auth.py Optional password authentication, signed cookies (~149 lines) - config.py Discovery, globals, model detection, reloadable config (~701 lines) - helpers.py HTTP helpers: j(), bad(), require(), safe_resolve(), security headers (~71 lines) - models.py Session model + CRUD, per-session profile tracking (~137 lines) - profiles.py Profile state management, hermes_cli wrapper (~246 lines) - onboarding.py First-run onboarding status, real provider config writes, and readiness detection. - routes.py All GET + POST route handlers (~1180 lines) - startup.py Startup helpers: auto_install_agent_deps() (~50 lines) - streaming.py SSE engine, run_agent, cancel, HERMES_HOME save/restore (~236 lines) - upload.py Multipart parser, file upload handler (~78 lines) - workspace.py File ops: list_dir, read_file_content, workspace helpers (~77 lines) + auth.py Optional password authentication, signed cookies (~366 lines) + config.py Discovery, globals, model detection, reloadable config (~4139 lines) + helpers.py HTTP helpers: j(), bad(), require(), safe_resolve(), security headers (~302 lines) + models.py Session model + CRUD, per-session profile tracking (~1927 lines) + profiles.py Profile state management, hermes_cli wrapper (~1056 lines) + onboarding.py First-run onboarding status, real provider config writes, OAuth linking, and readiness detection (~1002 lines) + routes.py All GET + POST route handlers (~9772 lines) + startup.py Startup helpers: auto_install_agent_deps() (~128 lines) + streaming.py SSE engine, run_agent, cancel, HERMES_HOME save/restore (~4420 lines) + upload.py Multipart parser, file upload handler (~284 lines) + workspace.py File ops: list_dir, read_file_content, workspace helpers (~810 lines) static/ - index.html HTML template (~364 lines) - style.css All CSS incl. mobile responsive (~670 lines) - ui.js DOM helpers, renderMd, tool cards, model dropdown, file tree (~977 lines) - workspace.js File preview, file ops, loadDir, clearPreview (~185 lines) - sessions.js Session CRUD, list rendering, search, SVG icons, dropdown actions (~533 lines) - messages.js send(), SSE event handlers, approval, transcript (~297 lines) - panels.js Cron, skills, memory, workspace, profiles, todo, settings (~974 lines) - commands.js Slash command registry, parser, autocomplete dropdown (~156 lines) + index.html HTML template (~1323 lines) + style.css All CSS incl. mobile responsive (~3767 lines) + ui.js DOM helpers, renderMd, tool cards, model dropdown, file tree (~7216 lines) + workspace.js File preview, file ops, loadDir, clearPreview (~369 lines) + sessions.js Session CRUD, list rendering, search, SVG icons, dropdown actions (~3517 lines) + messages.js send(), SSE event handlers, approval, transcript (~2301 lines) + panels.js Cron, skills, memory, workspace, profiles, todo, settings (~6480 lines) + commands.js Slash command registry, parser, autocomplete dropdown (~1302 lines) onboarding.js First-run wizard overlay, provider setup flow, and settings/workspace orchestration. - boot.js Event wiring, mobile sidebar/workspace nav, voice input, boot IIFE (~338 lines) + boot.js Event wiring, mobile sidebar/workspace nav, voice input, boot IIFE (~1607 lines) tests/ - conftest.py Isolated test server (port 8788, separate HERMES_HOME) (~240 lines) - test_sprint{1-20b}.py Feature tests per sprint (21 files, 415 test functions) - test_regressions.py Permanent regression gate (23 tests) - AGENTS.md Instruction file for agents working in this directory. + conftest.py Isolated test server/state fixtures (~644 lines) + 488 test files 5303 tests collected via pytest + test_regressions.py Permanent regression gate (~976 lines) + CONTRIBUTING.md Contributor workflow and PR expectations. ROADMAP.md Feature and product roadmap document. SPRINTS.md Forward sprint plan with CLI + Claude parity targets. ARCHITECTURE.md THIS FILE. @@ -90,7 +90,7 @@ actions. The topbar remains focused on conversation context and the workspace/fi State directory (runtime data, separate from source): - ~/.hermes/webui-mvp/ + ~/.hermes/webui/ sessions/ One JSON file per session: {session_id}.json workspaces.json Registered workspaces list last_workspace.txt Last-used workspace path @@ -99,7 +99,8 @@ State directory (runtime data, separate from source): Log file: - /tmp/webui-mvp.log stdout/stderr from the background server process + ~/.hermes/webui/bootstrap-8787.log start.sh/bootstrap background server log + ~/.hermes/webui.log ctl.sh daemon log --- @@ -118,15 +119,16 @@ Environment variables controlling behavior: HERMES_WEBUI_DEFAULT_WORKSPACE Default workspace path for new sessions HERMES_WEBUI_STATE_DIR Where sessions/ folder lives HERMES_CONFIG_PATH Path to ~/.hermes/config.yaml - HERMES_WEBUI_DEFAULT_MODEL Default LLM model string + HERMES_WEBUI_DEFAULT_MODEL Optional model override; unset means provider default HERMES_WEBUI_PASSWORD Optional: enable password auth (off by default) + HERMES_WEBUI_SKIP_ONBOARDING Optional: bypass the first-run onboarding wizard HERMES_HOME Base directory for Hermes state (~/.hermes by default) Test isolation environment variables (set by conftest.py): - HERMES_WEBUI_PORT=8788 Isolated test port - HERMES_WEBUI_STATE_DIR=~/.hermes/webui-mvp-test Isolated test state - HERMES_WEBUI_DEFAULT_WORKSPACE=.../test-workspace Isolated test workspace + HERMES_WEBUI_TEST_PORT=... Optional pinned test port + HERMES_WEBUI_TEST_STATE_DIR=~/.hermes/webui-test-* Optional pinned test state + HERMES_WEBUI_DEFAULT_WORKSPACE=.../test-workspace Isolated test workspace Tests NEVER talk to the production server (port 8787). The test state dir is wiped before each test session and deleted after. @@ -363,16 +365,18 @@ read_file_content(workspace, rel): ### 5.1 Structure The frontend is served from static/ as separate files: one HTML template, one CSS file, -and six JavaScript modules (~2,786 lines total). External dependencies: Prism.js (syntax -highlighting) and Mermaid.js (diagrams) from CDN, both loaded async/deferred with SRI hashes. - -Six JS modules loaded in order at end of : - 1. ui.js (~846 lines) DOM helpers, renderMd, tool card rendering, global state - 2. workspace.js (~169 lines) File tree, preview, file operations - 3. sessions.js (~532 lines) Session CRUD, list rendering, search, SVG icons, dropdown actions, project picker - 4. messages.js (~293 lines) send(), SSE event handlers, approval, transcript - 5. panels.js (~771 lines) Cron, skills, memory, workspace, todo, switchPanel - 6. boot.js (~175 lines) Event wiring + boot IIFE +and multiple JavaScript modules. External dependencies include Prism.js (syntax +highlighting), Mermaid.js (diagrams), xterm.js, and KaTeX assets loaded with the +current static template's integrity/CSP assumptions. + +Core JS modules loaded by the app include: + 1. ui.js (~7216 lines) DOM helpers, renderMd, tool card rendering, global state + 2. workspace.js (~369 lines) File tree, preview, file operations + 3. sessions.js (~3517 lines) Session CRUD, list rendering, search, SVG icons, dropdown actions, project picker + 4. messages.js (~2301 lines) send(), SSE event handlers, approval, transcript + 5. panels.js (~6480 lines) Cron, skills, memory, workspace, profiles, todo, settings + 6. commands.js (~1302 lines) Slash command registry, parser, autocomplete dropdown + 7. boot.js (~1607 lines) Event wiring + boot IIFE sessions.js defines an `ICONS` constant at module level with hardcoded SVG strings for all session action buttons (pin, unpin, folder, archive, unarchive, duplicate, trash). All icons @@ -680,27 +684,28 @@ Split server.py into a proper package. Completed across Sprints 4-10. Current structure: / - server.py Entry point + HTTP Handler dispatch (~76 lines) + server.py Entry point + HTTP Handler dispatch (~446 lines) api/ __init__.py - routes.py All GET + POST route handlers (~1016 lines) - config.py Configuration, constants, global state, model discovery (~640 lines) - helpers.py HTTP helpers: j(), bad(), require(), safe_resolve() (~57 lines) - models.py Session model + CRUD (~132 lines) - workspace.py File ops, workspace management (~77 lines) - upload.py Multipart parser, file upload handler (~77 lines) - streaming.py SSE engine, run_agent, cancel support (~222 lines) + routes.py All GET + POST route handlers (~9772 lines) + config.py Configuration, constants, global state, model discovery (~4139 lines) + helpers.py HTTP helpers: j(), bad(), require(), safe_resolve() (~302 lines) + models.py Session model + CRUD (~1927 lines) + workspace.py File ops, workspace management (~810 lines) + upload.py Multipart parser, file upload handler (~284 lines) + streaming.py SSE engine, run_agent, cancel support (~4420 lines) static/ index.html HTML document (served from disk) - style.css All CSS (~560 lines) - ui.js, workspace.js, sessions.js, messages.js, panels.js, boot.js + style.css All CSS (~3767 lines) + ui.js, workspace.js, sessions.js, messages.js, panels.js, commands.js, boot.js tests/ - conftest.py Isolated test server on port 8788 - test_sprint1-16.py Feature tests per sprint (14 files) + conftest.py Isolated test server/state fixtures + 488 test files 5303 tests collected test_regressions.py Permanent regression gate -Route extraction to api/routes.py completed in Sprint 11. server.py is now a ~76-line -thin shell: Handler class with structured logging, dispatch to routes, and main(). +Route extraction to api/routes.py completed in Sprint 11. server.py remains a +thin shell relative to the rest of the app: Handler class with headers, +structured logging, dispatch to routes, TLS wrapping, and main(). ### Phase B: Thread-Safe Request Context (Priority: Critical, Effort: Medium) @@ -779,7 +784,7 @@ Replacing with marked.js + DOMPurify is a future improvement (not blocking). ### Phase G: Observability -- MOSTLY COMPLETE -1. Structured JSON logging: COMPLETE (Sprint 1). Per-request JSON to /tmp/webui-mvp.log. +1. Structured JSON logging: COMPLETE (Sprint 1). Per-request JSON is printed to the active launcher log (`~/.hermes/webui/bootstrap-8787.log` for `start.sh`, `~/.hermes/webui.log` for `ctl.sh`). 2. Enhanced /health: COMPLETE (Sprint 7). Returns `active_streams`, `uptime_seconds`. 3. GET /api/debug/stats: NOT YET IMPLEMENTED. Low priority. @@ -795,13 +800,13 @@ Optional password gate for non-SSH-tunnel deployments. ### Phase I: Test Infrastructure -- COMPLETE -289 tests across 14 test files + regression gate. Isolated test server on port 8788 -with separate HERMES_HOME, wiped per run. Production data never touched. - -Test files: `test_sprint1.py` through `test_sprint11.py`, `test_sprint16.py`, `test_regressions.py`. -Fixtures in `conftest.py`: auto-cleanup, cron isolation, workspace reset. +5303 tests across 488 test files + regression gates. The pytest fixture derives +an isolated port and state directory from the repo path unless +`HERMES_WEBUI_TEST_PORT` / `HERMES_WEBUI_TEST_STATE_DIR` pin them explicitly. +Production data never touched. -Remaining: no CI (GitHub Actions), no frontend tests (browser-based). +Fixtures in `conftest.py`: auto-cleanup, profile/config isolation, cron +isolation, workspace reset, and test-server lifecycle. ### Phase J: Performance (Priority: Low, Effort: High) @@ -889,7 +894,8 @@ The api() helper: curl -s http://127.0.0.1:8787/health | python3 -m json.tool # Tail the server log live - tail -f /tmp/webui-mvp.log + tail -f ~/.hermes/webui/bootstrap-8787.log + tail -f ~/.hermes/webui.log # when launched through ctl.sh # List all sessions (metadata only) curl -s http://127.0.0.1:8787/api/sessions | python3 -m json.tool @@ -899,15 +905,15 @@ The api() helper: curl -s "http://127.0.0.1:8787/api/session?session_id=$SID" | python3 -m json.tool # Kill and restart server cleanly - pkill -f "python.*webui-mvp/server.py" - /webui-mvp/start.sh + pkill -f "python.*server.py" + /start.sh # Check if server process is running - ps aux | grep "webui-mvp/server.py" + ps aux | grep "server.py" # Inspect session files on disk - ls -lt ~/.hermes/webui-mvp/sessions/ - cat ~/.hermes/webui-mvp/sessions/SESSION_ID.json | python3 -m json.tool + ls -lt ~/.hermes/webui/sessions/ + cat ~/.hermes/webui/sessions/SESSION_ID.json | python3 -m json.tool # Count messages in a session python3 -c "import json; d=json.load(open('sessions/SID.json')); print(len(d['messages']))" @@ -920,9 +926,9 @@ The api() helper: curl -s http://127.0.0.1:8787/health # streams not exposed yet, add in Phase G # Find all sessions with messages (not Untitled empty) - ls ~/.hermes/webui-mvp/sessions/ | xargs -I{} python3 -c " + ls ~/.hermes/webui/sessions/ | xargs -I{} python3 -c " import json, sys - d = json.load(open('~/.hermes/webui-mvp/sessions/{}')) + d = json.load(open('~/.hermes/webui/sessions/{}')) if d['messages']: print('{}', d['title'][:50]) " 2>/dev/null @@ -1195,31 +1201,22 @@ will be working on this codebase. Read this before touching any file. ### Before Making Any Change 1. Read this document (ARCHITECTURE.md) fully. Especially sections 4, 5, and the ADRs. -2. Read the relevant section of server.py by searching for the SECTION header. +2. Inspect the relevant module under `api/` or `static/`; `server.py` is only the routing shell. 3. Check the Sprint Log (Section 15) to understand what was recently changed. -4. Run the test suite first to confirm baseline: cd && - venv/bin/python -m pytest webui-mvp/tests/test_sprint1.py -v +4. Run the relevant test slice first to confirm baseline, for example: + venv/bin/python -m pytest tests/test_regressions.py -q 5. Check server health: curl -s http://127.0.0.1:8787/health ### Making Changes -Always back up server.py before a non-trivial change: - cp server.py server.py.$(date +%Y%m%d_%H%M).bak - -Use exact string matching when patching. The pitfalls are documented in the -hermes-webui-mvp skill. Key ones: -- Never use sed on this file from the shell. Use execute_code with Python string replace. -- Always assert the old string is found before replacing (prevents silent no-op patches). -- Unicode escape sequences in JS (\u2026) exist as literal backslash-u in the file. - Match the file's raw content, not interpreted Python strings. -- The HTML block is a Python raw string (r"""..."""). Standard triple-quote escaping - rules do not apply inside it, but Python escape sequences \n etc. work in JS strings - inside it as literal two-character sequences. +Keep edits scoped to the module that owns the behavior. Use exact string +matching when making mechanical patches and verify that the intended old string +was found before replacing it. After any change: - venv/bin/python -m py_compile webui-mvp/server.py # syntax check + venv/bin/python -m py_compile server.py # syntax check curl -s http://127.0.0.1:8787/health # server still alive - venv/bin/python -m pytest webui-mvp/tests/ -v # tests still pass + venv/bin/python -m pytest tests/ -v # tests still pass ### Critical Rules (do NOT regress these) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a07d3e6661..ac02054706a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,34 @@ ## [Unreleased] +### Added + +- **PR #2162** by @franksong2702 — Refresh project-snapshot docs and add an explicit agent-onboarding entrypoint. Updates `AGENTS.md` (newly added), `README.md`, `ARCHITECTURE.md`, `TESTING.md`, `.env.example`, `.gitignore`, `docker_init.bash`, `docs/onboarding.md`, plus a new `docs/onboarding-agent-checklist.md`. Refreshes stale current-state claims (test counts, default model semantics, state/log paths, file inventory line counts), adds an explicit agent-safe install/test/run path for AI assistants doing human-assisted reinstalls, and adds `tests/test_docs_gitignore_policy.py` to pin the gitignore policy. Docs/test-only — no runtime behavior change. + +- **PR #2187** by @jasonjcwu (split from #2164) — Steer messages now appear in the chat transcript as semi-transparent italic user bubbles with a "Steer" badge while the turn is in flight, giving users clear feedback that their injected message reached the agent context. Previously, when `busy_input_mode` was `steer`, the injected message vanished entirely after a brief toast. The bubble is transient — the `done` event replaces `S.messages` with server state, so the badge disappears once the turn completes. CSS-only styling for the badge; no schema change. + +### Fixed + +- **PR #2171** by @franksong2702 — Session tail-window response (`/api/session?messages=1&resolve_model=0&msg_limit=30`) on long sessions is materially faster. Adds a cheap credential-marker prefilter before the full agent+fallback redaction pass — strings without known credential markers return immediately, while strings with likely markers still run the existing hard redaction. Skips the historical `session.tool_calls` list in the payload when returned messages already carry per-message tool metadata, avoiding sending the full historical list for every tail-window request. Same security and tool-card rendering behavior preserved. 173-line regression suite in `tests/test_session_tail_payload.py` + 81 LOC of new credential-prefilter tests in `tests/test_security_redaction.py`. + +- **PR #2182** by @LumenYoung — Compression banner no longer drifts away from the actual compaction boundary in long WebUI conversations. Fixes two related cases: (1) windowed transcript rendering — `renderMessages()` renders only a sliced `renderVisWithIdx` window, and when the compression anchor index wasn't found in the rendered window, the previous code passed the full visible index directly into the rendered-window array (usually out-of-window for long sessions), so the compression card fell back to `inner.appendChild(node)` and appeared near the newest messages instead of near the boundary; (2) persisted compaction reference messages — the `[CONTEXT COMPACTION — REFERENCE ONLY]` marker is now used as a stronger placement signal than anchor metadata when both are present. 60-line regression suite covering both cases. + +- **PR #2185** by @jasonjcwu — Switching sessions no longer surfaces a `Compression failed: not found` toast on the common case (no active compression). The `/api/session/compress/status` route handler was returning `None` from `j()` instead of `True`, so in edge cases (stale process state, exception during response write) the `do_GET` 404 fallback fired. Backend: `handle_get` now explicitly returns `True` after the status handler call. Frontend: `resumeManualCompressionForSession` catches 404 silently — no compression job means no error to surface. 139-line regression suite in `tests/test_compress_status_404_fix.py`. + +- **PR #2186** by @jasonjcwu (split from #2164) — Concurrent `send()` no longer drops user messages or swallows stream output. Two messages sent in rapid succession (queue drain + user click) could both pass the `S.busy` check because `setBusy(true)` only runs **after** the first `await` inside `send()`, leaving a window where two async `send()` calls ran concurrently. Adds a synchronous `_sendInProgress` flag at the very top of `send()` (before any `await`). Concurrent calls re-queue the message instead of silently dropping. `try/finally` ensures the flag resets on all exit paths. + +- **PR #2188** by @LumenYoung — Context progress ring refreshes immediately when automatic context compression completes. Previously the `compressed` SSE event only updated the compression card/toast; the context ring kept showing pre-compression token usage until a later `metering`/`done` update or the next message — making the UI look like compression had completed while the session was still near the limit. Backend now includes a live usage snapshot in the `compressed` SSE payload; frontend reads it and updates `S.lastUsage` + the composer context indicator atomically with the compression-card transition. + +- **PR #2189** by @xz-dev — Live metering usage updates are now scoped to the session currently visible in the chat pane. Pre-fix, background streams could overwrite `S.lastUsage` and the composer context indicator with metering data from a session the user wasn't looking at, making the indicator misleading on the active session. Four-line scope check inside the metering update path; no schema or SSE payload change. + +- **PR #2190** by @xz-dev — Thinking-card reasoning updates now update text in place during reasoning deltas instead of rebuilding the card DOM on every append. Preserves expand state and scroll position across reasoning streaming, so users reading a long reasoning block don't get bounced to the top on every chunk. When a thinking card exists, the update path now sets `pre.textContent` directly; full-rebuild path only fires when no existing card is present. + +### Stage-348 maintainer fixes + +- **`api/helpers.py:_SENSITIVE_LOWER_MARKERS` — add `"://"` URL marker** — Opus SHOULD-FIX-pre-merge on PR #2171's credential prefilter. The prefilter listed only specific DB scheme prefixes (`postgres://`, `mysql://`, `mongodb://`, `redis://`, `amqp://`) and a closed set of form keys (`token=`, `secret=`, `password=`, `authorization=`, `key=`), so OAuth callback URLs (`https://example.com/callback?code=AUTH_OPAQUE`), URL userinfo (`https://admin:supersecret@api.example.com/v1`), and signed-URL query params (`?signature=...`, `?session=...`) bypassed the hard agent redactor entirely — defeating the "WebUI API responses are a hard safety boundary" comment at `helpers.py:189`. Adding the generic `"://"` marker routes every http(s)/ws(s)/ftp URL to the hard redactor (which then selectively redacts only the sensitive substrings — plain `https://example.com/guide.html` URLs still pass through unchanged). Regression-pinned with 5 new parametric cases in `tests/test_security_redaction.py` (`test_redact_text_prefilter_covers_url_userinfo_and_sensitive_query_params`) covering OAuth code, URL userinfo, signed-URL signature, session query param, and WebSocket token — plus a negative-case `test_redact_text_prefilter_admits_plain_urls_without_sensitive_params` confirming the redactor doesn't over-redact plain URLs. Verified by reverting the fix locally: all 5 sensitive-URL cases fail; restoring the fix: all 5 pass. ~6 LOC code + ~50 LOC test. + +## [v0.51.54] — 2026-05-13 — Release AD (stage-347 — singleton self-built — NVIDIA NIM prefix preservation fix #2179) + ### Fixed - **PR #2179** (self-built, closes #2177) — NVIDIA NIM no longer 404s when WebUI is configured with `provider: nvidia` and a `nvidia/` id. `resolve_model_provider()` in `api/config.py` had the `_PORTAL_PROVIDERS` guard (Nous, OpenCode-Zen, OpenCode-Go, NVIDIA NIM — providers whose APIs require the full namespaced `provider/model` wire format) sitting **after** the `prefix == config_provider` strip branch. For `model_id="nvidia/nemotron-3-super-120b-a12b"` + `config_provider="nvidia"`, the strip branch fired first and returned the bare `nemotron-...` to NIM, which then 404'd because NIM requires the full path. Same bug class as #854 / #894 (Nous portal). The guard was originally added with NVIDIA in mind but the structural ordering was wrong. Fix is a pure reorder of two `if` blocks — hoist `_PORTAL_PROVIDERS` ahead of the strip — so all portal providers always preserve the full `provider/model` id regardless of whether the prefix happens to equal the provider name. Also closes a latent symmetric bug for the Nous case if a `nous/` id ever entered the catalog. Cross-tool trace against hermes-agent's `hermes_cli/models.py` (line 59, 177, 237-239) and `agent/model_metadata.py:68` confirms the agent CLI sends `nvidia/nemotron-...` verbatim — both tools now agree on the wire format. 118-line regression suite covering the reported case, cross-namespace `qwen/` and `meta/` ids, every static nvidia model in `_PROVIDER_MODELS`, the latent `nous/` ordering pin, and a non-portal-provider regression pin for the anthropic strip behavior. nesquena APPROVED with 200-line end-to-end trace + 12-shape behavioural harness + cross-tool wire-format verification. Reported on Discord by @vishnu in #report-bugs. diff --git a/README.md b/README.md index f7563c6c143..2539daff407 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ The bootstrap will: If provider setup is still incomplete after install, the onboarding wizard will point you to finish it with `hermes model` instead of trying to replicate the full CLI setup in-browser. For a step-by-step walkthrough of the wizard, provider choices, local model server Base URLs, and safe re-runs, see [`docs/onboarding.md`](docs/onboarding.md). +If an AI assistant is helping with install, reinstall, bootstrap, provider setup, or first-run support, have it read [`docs/onboarding-agent-checklist.md`](docs/onboarding-agent-checklist.md) before running commands or inspecting logs. --- @@ -267,7 +268,7 @@ Full list of environment variables: | `HERMES_WEBUI_PORT` | `8787` | Port | | `HERMES_WEBUI_STATE_DIR` | `~/.hermes/webui` | Where sessions and state are stored | | `HERMES_WEBUI_DEFAULT_WORKSPACE` | `~/workspace` | Default workspace | -| `HERMES_WEBUI_DEFAULT_MODEL` | `openai/gpt-5.4-mini` | Default model | +| `HERMES_WEBUI_DEFAULT_MODEL` | *(provider default)* | Optional model override; leave unset to use the active Hermes provider default | | `HERMES_WEBUI_PASSWORD` | *(unset)* | Set to enable password authentication | | `HERMES_WEBUI_EXTENSION_DIR` | *(unset)* | Optional local directory served at `/extensions/`; must point to an existing directory before extension injection is enabled | | `HERMES_WEBUI_EXTENSION_SCRIPT_URLS` | *(unset)* | Optional comma-separated same-origin script URLs to inject; see [WebUI Extensions](docs/EXTENSIONS.md) | @@ -367,9 +368,9 @@ Or using the agent venv explicitly: /path/to/hermes-agent/venv/bin/python -m pytest tests/ -v ``` -Tests run against an isolated server on port 8788 with a separate state directory. -Production data and real cron jobs are never touched. Current count: **3309 tests** -across 100+ test files. +Tests run against an isolated server with a separate state directory. +Production data and real cron jobs are never touched. Current snapshot: +**5303 tests collected** across **488 test files**. --- @@ -491,33 +492,33 @@ across 100+ test files. ## Architecture ``` -server.py HTTP routing shell + auth middleware (~154 lines) +server.py HTTP routing shell + auth middleware (~446 lines) api/ - auth.py Optional password authentication, signed cookies (~201 lines) - config.py Discovery, globals, model detection, reloadable config (~1110 lines) - helpers.py HTTP helpers, security headers (~175 lines) - models.py Session model + CRUD + CLI bridge (~377 lines) - onboarding.py First-run onboarding wizard, OAuth provider support (~507 lines) - profiles.py Profile state management, hermes_cli wrapper (~411 lines) - routes.py All GET + POST route handlers (~2250 lines) - state_sync.py /insights sync — message_count to state.db (~113 lines) - streaming.py SSE engine, run_agent, cancel support (~660 lines) - updates.py Self-update check and release notes (~257 lines) - upload.py Multipart parser, file upload handler (~82 lines) - workspace.py File ops, workspace helpers, git detection (~288 lines) + auth.py Optional password authentication, signed cookies (~366 lines) + config.py Discovery, globals, model detection, reloadable config (~4139 lines) + helpers.py HTTP helpers, security headers (~302 lines) + models.py Session model + CRUD + CLI bridge (~1927 lines) + onboarding.py First-run onboarding wizard, OAuth provider support (~1002 lines) + profiles.py Profile state management, hermes_cli wrapper (~1056 lines) + routes.py All GET + POST route handlers (~9772 lines) + state_sync.py /insights sync — message_count to state.db (~118 lines) + streaming.py SSE engine, run_agent, cancel support (~4420 lines) + updates.py Self-update check and release notes (~545 lines) + upload.py Multipart parser, file upload handler (~284 lines) + workspace.py File ops, workspace helpers, git detection (~810 lines) static/ - index.html HTML template (~600 lines) - style.css All CSS incl. mobile responsive, themes (~1050 lines) - ui.js DOM helpers, renderMd, tool cards, context indicator (~1740 lines) - workspace.js File preview, file ops, git badge (~286 lines) - sessions.js Session CRUD, collapsible groups, search, reload recovery (~800 lines) - messages.js send(), SSE handlers, live streaming, session recovery (~655 lines) - panels.js Cron, skills, memory, profiles, settings (~1438 lines) - commands.js Slash command autocomplete (~267 lines) - boot.js Mobile nav, voice input, boot IIFE (~524 lines) + index.html HTML template (~1323 lines) + style.css All CSS incl. mobile responsive, themes (~3767 lines) + ui.js DOM helpers, renderMd, tool cards, context indicator (~7216 lines) + workspace.js File preview, file ops, git badge (~369 lines) + sessions.js Session CRUD, collapsible groups, search, reload recovery (~3517 lines) + messages.js send(), SSE handlers, live streaming, session recovery (~2301 lines) + panels.js Cron, skills, memory, profiles, settings (~6480 lines) + commands.js Slash command autocomplete (~1302 lines) + boot.js Mobile nav, voice input, boot IIFE (~1607 lines) tests/ - conftest.py Isolated test server (port 8788) - 61 test files 961 test functions + conftest.py Isolated test server/state fixtures + 488 test files 5303 tests collected Dockerfile python:3.12-slim container image docker-compose.yml Compose with named volume and optional auth .github/workflows/ CI: multi-arch Docker build + GitHub Release on tag @@ -537,8 +538,14 @@ State lives outside the repo at `~/.hermes/webui/` by default - `CHANGELOG.md` -- release notes per sprint - `SPRINTS.md` -- forward sprint plan with CLI + Claude parity targets - `THEMES.md` -- theme system documentation, custom theme guide +- `docs/docker.md` -- Docker compose setup, common failures, and bind-mount migration +- `docs/supervisor.md` -- launchd, systemd, supervisord, runit, and s6 process-supervisor setup - `docs/onboarding.md` -- first-run wizard, provider setup, local model server Base URLs, and safe re-runs +- `docs/onboarding-agent-checklist.md` -- safety rules, evidence commands, and pass/fail checks for assistant-led install or reinstall support - `docs/troubleshooting.md` -- diagnostic flows for common failures (e.g. "AIAgent not available") +- `docs/wsl-autostart.md` -- WSL2 auto-start at Windows login +- `docs/EXTENSIONS.md` -- administrator-controlled WebUI extension injection +- `docs/rfcs/README.md` -- RFC index for larger architecture and durability proposals ## Contributors diff --git a/TESTING.md b/TESTING.md index ee35af45b9a..b472570aff7 100644 --- a/TESTING.md +++ b/TESTING.md @@ -8,7 +8,7 @@ > Prerequisites: SSH tunnel is active on port 8787. Open http://localhost:8787 in browser. > Server health check: curl http://127.0.0.1:8787/health should return {"status":"ok"}. > -> Automated coverage: 3648 tests collected via `pytest tests/ --collect-only -q`. Tests run on every PR via GitHub Actions on Python 3.11, 3.12, and 3.13. The suite covers the bootstrap/static wizard, real provider config persistence (`config.yaml` + `.env`), the `/api/onboarding/*` backend, the onboarding skip/existing-config guard, CSS regression coverage for thinking/tool card animation, streaming session persistence, mobile layout breakpoints, locale parity across 9 languages, and ~700 issue/PR-pinned regression tests. +> Automated coverage: 5303 tests collected via `pytest tests/ --collect-only -q`. Tests run on every PR via GitHub Actions on Python 3.11, 3.12, and 3.13. The suite covers the bootstrap/static wizard, real provider config persistence (`config.yaml` + `.env`), the `/api/onboarding/*` backend, the onboarding skip/existing-config guard, CSS regression coverage for thinking/tool card animation, streaming session persistence, mobile layout breakpoints, locale parity across 11 languages, and hundreds of issue/PR-pinned regression tests. > Run: `pytest tests/ -v --timeout=60` > > Local regression focus: verify that a previously closed workspace panel stays visually closed from first paint through boot completion on desktop refresh; there should be no brief open-then-close flash. @@ -533,7 +533,8 @@ FAIL: Sidebar causes layout overflow or blocks chat. ### T11.3: Structured Log Output SETUP: SSH access to the server. STEPS: - 1. In a terminal: tail -f /tmp/webui-mvp.log + 1. In a terminal: tail -f ~/.hermes/webui/bootstrap-8787.log + (or tail -f ~/.hermes/webui.log when launched through `ctl.sh`) 2. In browser: perform any action (load page, send message, click file) EXPECT: - Log entries appear in terminal as JSON: {"ts":"...","method":"GET","path":"/health","status":200,"ms":0.1} @@ -577,7 +578,7 @@ FAIL: Browser freezes, crash, or security issue. ## Automated Test Coverage Reference -These behaviors are verified by pytest (run: venv/bin/python -m pytest webui-mvp/tests/ -v): +These behaviors are verified by pytest (run: venv/bin/python -m pytest tests/ -v): Sprint 1 tests (test_sprint1.py): - Server health, session CRUD (create/load/update/delete/sort) @@ -1835,8 +1836,8 @@ Bridged CLI sessions: --- -*Last updated: v0.51.31, May 9, 2026* -*Total automated tests collected: 4977* +*Last updated: v0.51.54, May 13, 2026* +*Total automated tests collected: 5303* *Regression gate: tests/test_regressions.py* *Run: pytest tests/ -v --timeout=60* *Source: /* diff --git a/api/helpers.py b/api/helpers.py index 7cf010c7663..76bc404af4f 100644 --- a/api/helpers.py +++ b/api/helpers.py @@ -112,7 +112,8 @@ def t(handler, payload, status: int=200, content_type: str='text/plain; charset= def _build_redact_fn(): """Return a redactor backed by hermes-agent plus local fallback patterns.""" - # Minimal fallback covering the most common credential prefixes. + # Fallback mirrors the agent's known credential prefixes so WebUI API + # responses remain a hard redaction boundary even without hermes-agent. # Keep this active even when hermes-agent is importable so API responses do # not regress if the agent redactor misses a token shape. _CRED_RE = _re.compile( @@ -124,10 +125,34 @@ def _build_redact_fn(): r"|ghu_[A-Za-z0-9]{10,}" # GitHub user-to-server token r"|ghs_[A-Za-z0-9]{10,}" # GitHub server-to-server token r"|ghr_[A-Za-z0-9]{10,}" # GitHub refresh token + r"|xox[baprs]-[A-Za-z0-9-]{10,}" # Slack tokens + r"|AIza[A-Za-z0-9_-]{30,}" # Google API keys + r"|pplx-[A-Za-z0-9]{10,}" # Perplexity + r"|fal_[A-Za-z0-9_-]{10,}" # Fal.ai + r"|fc-[A-Za-z0-9]{10,}" # Firecrawl + r"|bb_live_[A-Za-z0-9_-]{10,}" # BrowserBase + r"|gAAAA[A-Za-z0-9_=-]{20,}" # Codex encrypted tokens r"|AKIA[A-Z0-9]{16}" # AWS Access Key ID - r"|xox[baprs]-[A-Za-z0-9-]{10,}" # Slack tokens - r"|hf_[A-Za-z0-9]{10,}" # HuggingFace token - r"|SG\.[A-Za-z0-9_-]{10,}" # SendGrid API key + r"|sk_live_[A-Za-z0-9]{10,}" # Stripe secret key (live) + r"|sk_test_[A-Za-z0-9]{10,}" # Stripe secret key (test) + r"|rk_live_[A-Za-z0-9]{10,}" # Stripe restricted key + r"|SG\.[A-Za-z0-9_-]{10,}" # SendGrid API key + r"|hf_[A-Za-z0-9]{10,}" # HuggingFace token + r"|r8_[A-Za-z0-9]{10,}" # Replicate API token + r"|npm_[A-Za-z0-9]{10,}" # npm access token + r"|pypi-[A-Za-z0-9_-]{10,}" # PyPI API token + r"|dop_v1_[A-Za-z0-9]{10,}" # DigitalOcean PAT + r"|doo_v1_[A-Za-z0-9]{10,}" # DigitalOcean OAuth + r"|am_[A-Za-z0-9_-]{10,}" # AgentMail API key + r"|sk_[A-Za-z0-9_]{10,}" # ElevenLabs TTS key + r"|tvly-[A-Za-z0-9]{10,}" # Tavily search API key + r"|exa_[A-Za-z0-9]{10,}" # Exa search API key + r"|gsk_[A-Za-z0-9]{10,}" # Groq Cloud API key + r"|syt_[A-Za-z0-9]{10,}" # Matrix access token + r"|retaindb_[A-Za-z0-9]{10,}" # RetainDB API key + r"|hsk-[A-Za-z0-9]{10,}" # Hindsight API key + r"|mem0_[A-Za-z0-9]{10,}" # Mem0 Platform API key + r"|brv_[A-Za-z0-9]{10,}" # ByteRover API key r")(?![A-Za-z0-9_-])" ) _AUTH_HDR_RE = _re.compile(r"(Authorization:\s*Bearer\s+)(\S+)", _re.IGNORECASE) @@ -179,6 +204,103 @@ def _combined_redact(text: str) -> str: _redact_fn_cached = _build_redact_fn() +_SENSITIVE_CASE_MARKERS = ( + "sk-", + "ghp_", + "github_pat_", + "gho_", + "ghu_", + "ghs_", + "ghr_", + "AKIA", + "xoxb-", + "xoxa-", + "xoxp-", + "xoxr-", + "xoxs-", + "AIza", + "pplx-", + "fal_", + "fc-", + "bb_live_", + "gAAAA", + "sk_live_", + "sk_test_", + "rk_live_", + "SG.", + "hf_", + "r8_", + "npm_", + "pypi-", + "dop_v1_", + "doo_v1_", + "am_", + "sk_", + "tvly-", + "exa_", + "gsk_", + "syt_", + "retaindb_", + "hsk-", + "mem0_", + "brv_", + "eyJ", + "-----BEGIN", +) +_SENSITIVE_LOWER_MARKERS = ( + "authorization: bearer ", + "private key", + "postgres://", + "postgresql://", + "mysql://", + "mongodb://", + "redis://", + "amqp://", + "://", # stage-348 Opus SHOULD-FIX: catch http(s)/ws(s)/ftp URL userinfo + sensitive query params (#2171 follow-up) + "access_token", + "refresh_token", + "id_token", + "api_key", + "apikey", + "client_secret", + "auth_token", + "raw_secret", + "secret_input", + "key_material", + "x-amz-signature", + "token=", + "secret=", + "password=", + "authorization=", + "key=", + '"token"', + '"secret"', + '"password"', + '"bearer"', +) +_SENSITIVE_TELEGRAM_MARKER_RE = _re.compile(r"(?:bot)?\d{8,}:[-A-Za-z0-9_]{30,}") +_SENSITIVE_DISCORD_MARKER_RE = _re.compile(r"<@!?\d{17,20}>") +_SENSITIVE_PHONE_MARKER_RE = _re.compile(r"(? bool: + """Cheap prefilter before the full agent+fallback redaction pass.""" + if not isinstance(text, str) or not text: + return False + if any(marker in text for marker in _SENSITIVE_CASE_MARKERS): + return True + lower = text.lower() + if any(marker in lower for marker in _SENSITIVE_LOWER_MARKERS): + return True + if ":" in text and _SENSITIVE_TELEGRAM_MARKER_RE.search(text): + return True + if "<@" in text and _SENSITIVE_DISCORD_MARKER_RE.search(text): + return True + if "+" in text and _SENSITIVE_PHONE_MARKER_RE.search(text): + return True + return False + + def _redact_text(text: str, *, _enabled: bool | None = None) -> str: """Redact sensitive text from API responses. Respects api_redact_enabled setting. @@ -194,6 +316,8 @@ def _redact_text(text: str, *, _enabled: bool | None = None) -> str: _enabled = bool(load_settings().get("api_redact_enabled", True)) if not _enabled: return text + if not _might_contain_sensitive_text(text): + return text return _redact_fn_cached(text) diff --git a/api/routes.py b/api/routes.py index df290280920..76359aa1f6c 100644 --- a/api/routes.py +++ b/api/routes.py @@ -1666,6 +1666,24 @@ def _is_messaging_session_record(session) -> bool: return _is_known_messaging_source(raw) +def _messages_include_tool_metadata(messages) -> bool: + """Return true when returned messages can reconstruct their own tool cards.""" + if not isinstance(messages, list): + return False + for msg in messages: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + if isinstance(msg.get("tool_calls"), list) and msg.get("tool_calls"): + return True + content = msg.get("content") + if isinstance(content, list) and any( + isinstance(part, dict) and part.get("type") == "tool_use" + for part in content + ): + return True + return False + + def _is_messaging_session_id(sid: str) -> bool: """Detect messaging-backed sessions from WebUI metadata or Agent rows.""" try: @@ -3195,7 +3213,8 @@ def handle_get(handler, parsed) -> bool: if parsed.path == "/api/session/compress/status": query = parse_qs(parsed.query) - return _handle_session_compress_status(handler, query.get("session_id", [""])[0]) + _handle_session_compress_status(handler, query.get("session_id", [""])[0]) + return True if parsed.path == "/api/session": import time as _time @@ -3362,9 +3381,19 @@ def handle_get(handler, parsed) -> bool: _persisted_cl = _fb_cl except Exception: pass + _session_tool_calls = getattr(s, "tool_calls", []) if load_messages else [] + if ( + load_messages + and msg_limit is not None + and _messages_include_tool_metadata(_truncated_msgs) + ): + # The browser ignores session-level tool_calls when the returned + # messages already carry per-message tool metadata. Avoid sending + # the full historical list with a small tail window. + _session_tool_calls = [] raw = s.compact() | { "messages": _truncated_msgs, - "tool_calls": getattr(s, "tool_calls", []) if load_messages else [], + "tool_calls": _session_tool_calls, "active_stream_id": getattr(s, "active_stream_id", None), "pending_user_message": getattr(s, "pending_user_message", None), "pending_attachments": getattr(s, "pending_attachments", []) if load_messages else [], diff --git a/api/streaming.py b/api/streaming.py index ff1e4a0773b..c596f2bfa26 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -3421,7 +3421,9 @@ def _periodic_checkpoint(): or _compression_summary_from_messages(s.context_messages) ) put('compressed', { + 'session_id': s.session_id, 'message': 'Context auto-compressed to continue the conversation', + 'usage': _live_usage_snapshot(), }) # Stamp 'timestamp' on any messages that don't have one yet diff --git a/docker_init.bash b/docker_init.bash index fbe71780c1b..b45eb0b63a1 100644 --- a/docker_init.bash +++ b/docker_init.bash @@ -277,7 +277,7 @@ rm -f $it || error_exit "Failed to delete test file in /app" echo ""; echo "== Checking required environment variables for hermes-webui" -echo ""; echo "-- HERMES_WEBUI_VERSION: Where to store sessions, workspaces, and other state (default: ~/.hermes/webui-mvp)" +echo ""; echo "-- HERMES_WEBUI_STATE_DIR: Where to store sessions, workspaces, and other state (default: ~/.hermes/webui)" if [ -z "${HERMES_WEBUI_STATE_DIR+x}" ]; then error_exit "HERMES_WEBUI_STATE_DIR not set"; fi; echo "-- HERMES_WEBUI_STATE_DIR: $HERMES_WEBUI_STATE_DIR" if [ ! -d "$HERMES_WEBUI_STATE_DIR" ]; then mkdir -p $HERMES_WEBUI_STATE_DIR || error_exit "Failed to create state directory at $HERMES_WEBUI_STATE_DIR"; fi diff --git a/docs/onboarding-agent-checklist.md b/docs/onboarding-agent-checklist.md new file mode 100644 index 00000000000..df62f90e867 --- /dev/null +++ b/docs/onboarding-agent-checklist.md @@ -0,0 +1,207 @@ +# Agent-assisted onboarding checklist + +This checklist is for an AI assistant helping a human install, reinstall, or +debug Hermes WebUI onboarding. It does not replace the human first-run wizard. +Use it before running bootstrap commands, inspecting logs, or recommending a +cleanup path. + +If you are an AI assistant, read this file before assisting with onboarding, +bootstrap, provider setup, reinstall, or first-run support. + +## Role split + +The human operator owns: + +- choosing the install path +- choosing the provider and model +- entering API keys, OAuth codes, and passwords +- approving any cleanup of a real Hermes home +- approving any external exposure outside localhost + +The assistant owns: + +- using isolated trial directories unless the human explicitly says otherwise +- checking non-secret status endpoints and logs +- explaining which step passed or failed +- collecting redacted evidence for Discord or GitHub support +- stopping before destructive cleanup, credential handling, or public exposure + +## Hard safety rules + +- Do not delete, move, or overwrite the real `~/.hermes` directory unless the + human explicitly asks for that exact action. +- Do not print API keys, OAuth tokens, cookies, full `.env` files, full + `auth.json` files, or password hashes. +- Do not modify real cron jobs, real sessions, real profiles, or real memory + files during an onboarding trial. +- Do not expose WebUI on a public interface without password protection and + explicit human approval. +- Do not proxy or tunnel local service checks such as `localhost`, + `127.0.0.1`, private LAN addresses, or Docker container loopback paths. + +## Pre-flight + +Confirm the basic context: + +```bash +pwd +git branch --show-current +git rev-parse --short HEAD +python3 --version +``` + +Check whether repo-local environment overrides will affect bootstrap: + +```bash +test -f .env && grep -n 'HERMES_HOME\|HERMES_WEBUI_STATE_DIR\|HERMES_WEBUI_PORT\|HERMES_WEBUI_HOST' .env +``` + +If `.env` exists, do not print the full file. Inspect only the specific +non-secret keys needed to understand the active Hermes home, WebUI state +directory, port, or host. + +## Isolated local trial + +Use an isolated Hermes home and WebUI state directory for a reinstall or support +trial. This keeps the test away from the operator's real memory, sessions, +profiles, credentials, and cron state. + +```bash +mkdir -p ~/hermes-onboarding-test +HERMES_HOME=~/hermes-onboarding-test/.hermes \ +HERMES_WEBUI_STATE_DIR=~/hermes-onboarding-test/webui \ +HERMES_WEBUI_PORT=8789 \ +python3 bootstrap.py +``` + +Open: + +```text +http://127.0.0.1:8789 +``` + +The bootstrap writes a port-specific log under the selected WebUI state +directory: + +```text +~/hermes-onboarding-test/webui/bootstrap-8789.log +``` + +For daemon-style installs, `ctl.sh` writes the daemon log to the active +`HERMES_HOME` by default: + +```text +~/.hermes/webui.log +``` + +When using the isolated trial environment, prefer the bootstrap command above +unless the human specifically wants to validate `ctl.sh`. + +## Non-secret evidence commands + +After the server starts, collect status without secrets: + +```bash +curl -sS http://127.0.0.1:8789/health +curl -sS http://127.0.0.1:8789/api/onboarding/status +find ~/hermes-onboarding-test -maxdepth 3 -type f | sort +tail -n 120 ~/hermes-onboarding-test/webui/bootstrap-8789.log +``` + +When summarizing `/api/onboarding/status`, focus on: + +- `completed` +- `system.hermes_found` +- `system.imports_ok` +- `system.config_path` +- `system.config_exists` +- `system.setup_state` +- `system.provider_configured` +- `system.provider_ready` +- `system.chat_ready` +- `system.current_provider` +- `system.current_model` +- `system.current_base_url` +- `system.env_path` + +Do not paste the full payload if it contains unexpected sensitive local paths +or values. Redact paths and provider details when the human asks for a public +GitHub or Discord support report. + +## Pass criteria + +A local onboarding trial passes when: + +- `/health` returns successfully. +- `/api/onboarding/status` returns JSON. +- The wizard appears when `completed` is false. +- The wizard stays out of the way when `completed` is true or + `HERMES_WEBUI_SKIP_ONBOARDING=1` is intentionally set. +- `system.hermes_found` and `system.imports_ok` match the expected bootstrap + state. +- `system.provider_ready` and `system.chat_ready` become true after the human + completes a provider path that should support chat. +- `system.config_path` and `system.env_path` point inside the intended isolated + `HERMES_HOME` during a trial. +- WebUI files are written under the intended `HERMES_WEBUI_STATE_DIR`. + +If the human chooses a provider that must be completed in the CLI, passing can +mean the wizard correctly points them to `hermes model` or `hermes auth` rather +than trying to collect unsupported credentials in the browser. + +## Failure triage + +If the server does not start: + +- check the bootstrap log +- check for a port conflict on `8789` +- confirm Python can run `bootstrap.py` +- confirm `.env` is not overriding the isolated directories or port + +If onboarding reports `agent_unavailable`: + +- confirm the bootstrap found or installed Hermes Agent +- check whether the running Python can import `run_agent.AIAgent` +- use `docs/troubleshooting.md`, especially the `AIAgent not available` flow + +If onboarding reports `provider_incomplete`: + +- confirm whether the provider is API-key based, OAuth based, or local +- let the human enter credentials or run the CLI auth flow +- do not ask the human to paste secrets into chat + +If a local model server does not probe successfully: + +- from native macOS/Linux, use `http://127.0.0.1:/v1` when the server is + on the same host +- from Docker Desktop, use `http://host.docker.internal:/v1` +- from another LAN machine, use the server's LAN IP and `/v1` +- remember that `localhost` inside a container is the container itself + +If password or reverse-proxy behavior is confusing: + +- keep the first pass on `127.0.0.1` +- require password protection before exposing WebUI beyond localhost +- include the reverse proxy shape in the support report without pasting tokens + or cookies + +## Final support report + +Use this shape when reporting results to the human, Discord, or GitHub: + +```text +Install path: +OS / Python: +Repo commit: +Command used: +WebUI URL: +State isolation: +Health result: +Onboarding status summary: +Files created or changed: +Log excerpt: +Pass/fail: +Next recommended action: +``` + +Redact secrets and private paths before posting publicly. diff --git a/docs/onboarding.md b/docs/onboarding.md index f6409f96a8e..b53b68fc85e 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -3,6 +3,11 @@ This guide explains what happens the first time Hermes WebUI starts, which setup path to choose, and how to recover when the wizard cannot finish. +If an AI assistant is helping with install, reinstall, bootstrap, provider +setup, or first-run support, read +[`docs/onboarding-agent-checklist.md`](onboarding-agent-checklist.md) before +running commands or inspecting logs. + The short version: run the bootstrap, open the WebUI, choose a provider, choose a workspace, optionally set a password, then start a chat. If you are using a local model server from Docker, pay special attention to the Base URL section @@ -55,6 +60,10 @@ python3 bootstrap.py Then open `http://127.0.0.1:8789`. +For an assistant-led trial run, follow the safety rules, evidence commands, and +pass/fail criteria in +[`docs/onboarding-agent-checklist.md`](onboarding-agent-checklist.md). + If your repo has a `.env` file, remember that the bootstrap loads it. Remove or adjust any `HERMES_HOME`, `HERMES_WEBUI_STATE_DIR`, or `HERMES_WEBUI_PORT` entries there before using the isolated command above. diff --git a/static/commands.js b/static/commands.js index c09cbf54f19..6a2ac260556 100644 --- a/static/commands.js +++ b/static/commands.js @@ -484,6 +484,9 @@ async function resumeManualCompressionForSession(sid){ if(!S.session||S.session.session_id!==sid) return; await _applyManualCompressionResult(done, status.focus_topic||'', visibleCount, status.focus_topic?`/compress ${status.focus_topic}`:'/compress'); }catch(e){ + // No active compression job or transient server error — not a real failure. + // 404: route missed or session gone; 5xx: backend exception during status check. + if(e&&(!e.status||e.status===404||e.status>=500)) return; if(S.session&&S.session.session_id===sid&&typeof setCompressionUi==='function'){ const visibleMessages=_manualCompressionVisibleMessages(); setCompressionUi({ @@ -875,6 +878,26 @@ async function cmdSteer(args){ * @param {boolean} explicitSteer - True if the user explicitly invoked /steer * (vs the busy-mode auto-fallback). Affects toast wording only. */ +function _showSteerIndicator(text){ + const inner=document.getElementById('msgInner'); + if(!inner) return; + // Remove any existing steer indicator + const old=inner.querySelector('.steer-indicator'); + if(old) old.remove(); + const el=document.createElement('div'); + el.className='steer-indicator'; + const badge=document.createElement('span'); + badge.className='steer-badge'; + badge.textContent='Steer'; + const body=document.createElement('span'); + body.className='steer-body'; + body.textContent=text.length>120?text.slice(0,117)+'…':text; + el.appendChild(badge); + el.appendChild(body); + inner.appendChild(el); + if(typeof scrollToBottom==='function') scrollToBottom(); +} + async function _trySteer(msg, explicitSteer){ let result=null; try{ @@ -887,6 +910,11 @@ async function _trySteer(msg, explicitSteer){ result={accepted:false, fallback:'network_error'}; } if(result&&result.accepted){ + // Show a transient steer indicator in the chat (NOT in S.messages — it must + // survive the done event's S.messages=d.session.messages replacement). + // The indicator self-removes when the turn completes (done/cancel/error + // all call renderMessages which rebuilds msgInner). + _showSteerIndicator(msg); showToast(t('cmd_steer_delivered'),2500); return; } diff --git a/static/messages.js b/static/messages.js index fc7c6d975fb..ddf13d5ad9a 100644 --- a/static/messages.js +++ b/static/messages.js @@ -52,7 +52,28 @@ const _msgEl=document.getElementById('msg'); if(_msgEl) _msgEl.addEventListener('focus', ()=>{ if('speechSynthesis' in window && speechSynthesis.speaking) speechSynthesis.pause(); }); if(_msgEl) _msgEl.addEventListener('blur', ()=>{ if('speechSynthesis' in window && speechSynthesis.paused) speechSynthesis.resume(); }); +// Guard against concurrent send() calls. Without this, two rapid sends +// (e.g. queue drain + user click) can both pass the S.busy check because +// setBusy(true) is only called after the first await inside send(). +let _sendInProgress = false; + async function send(){ + // Reject concurrent invocations early — before any await yields control. + // If a send is already in-flight (e.g. queue drain), re-queue the message + // instead of silently dropping it. + if (_sendInProgress) { + const _text=$('msg').value.trim(); + if(_text && S.session && S.session.session_id){ + queueSessionMessage(S.session.session_id,{text:_text,files:[...S.pendingFiles],model:S.session&&S.session.model||($('modelSelect')&&$('modelSelect').value)||'',model_provider:S.session&&S.session.model_provider||null,profile:S.activeProfile||'default'}); + $('msg').value='';autoResize(); + S.pendingFiles=[];renderTray(); + updateQueueBadge(S.session.session_id); + showToast(`Queued: "${_text.slice(0,40)}${_text.length>40?'…':''}"`,2000); + } + return; + } + _sendInProgress = true; + try{ const text=$('msg').value.trim(); if(!text&&!S.pendingFiles.length)return; // Don't send while an inline message edit is active @@ -337,6 +358,7 @@ async function send(){ // Open SSE stream and render tokens live attachLiveStream(activeSid, streamId, uploadedNames); + }finally{ _sendInProgress=false; } } const LIVE_STREAMS={}; @@ -1175,7 +1197,12 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ if(!S.session||S.session.session_id!==activeSid) return; let d={}; try{ d=JSON.parse(e.data||'{}')||{}; }catch(_){ d={}; } + if(d.session_id&&d.session_id!==activeSid) return; const message=String(d.message||'Context auto-compressed to continue the conversation').trim(); + if(d.usage&&typeof _syncCtxIndicator==='function'){ + S.lastUsage={...(S.lastUsage||{}),...d.usage}; + _syncCtxIndicator(S.lastUsage); + } if(typeof setCompressionUi==='function'){ setCompressionUi({ sessionId:activeSid, @@ -1195,8 +1222,10 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ const d=JSON.parse(e.data||'{}'); if((d.session_id||activeSid)!==activeSid) return; if(d.usage&&typeof _syncCtxIndicator==='function'){ - S.lastUsage={...(S.lastUsage||{}),...d.usage}; - _syncCtxIndicator(S.lastUsage); + if(S.session&&S.session.session_id===activeSid){ + S.lastUsage={...(S.lastUsage||{}),...d.usage}; + _syncCtxIndicator(S.lastUsage); + } } if(d.estimated===true||d.tps_available!==true||typeof d.tps!=='number'||d.tps<=0){ if(typeof _setLiveAssistantTps==='function') _setLiveAssistantTps(null); diff --git a/static/style.css b/static/style.css index 9866171825a..7b463d4cea5 100644 --- a/static/style.css +++ b/static/style.css @@ -796,6 +796,10 @@ @media(min-width:1800px){.messages-inner{max-width:1200px;}} .msg-row{padding:10px 0;} .msg-row+.msg-row{border-top:none;} + /* Steer indicator: transient banner below messages, removed on renderMessages rebuild */ + .steer-indicator{display:flex;align-items:baseline;gap:8px;padding:10px 0;opacity:.65;font-style:italic;color:var(--accent-text);} + .steer-indicator .steer-body{white-space:pre-wrap;word-break:break-word;} + .steer-badge{display:inline-block;font-size:10px;font-weight:600;letter-spacing:.04em;text-transform:uppercase;color:var(--accent-text);background:var(--accent-bg);border:1px solid var(--accent-bg-strong);border-radius:4px;padding:1px 6px;vertical-align:middle;line-height:1.6;font-style:normal;flex-shrink:0;} .msg-role{font-size:12px;font-weight:500;letter-spacing:.01em;margin-bottom:8px;display:flex;align-items:center;gap:8px;} .msg-role.user{color:var(--accent);} .msg-role.assistant{color:var(--accent-text);opacity:.6;} diff --git a/static/ui.js b/static/ui.js index 2e9a7da51ce..6d557ccaf9a 100644 --- a/static/ui.js +++ b/static/ui.js @@ -4472,9 +4472,11 @@ function _compressionAnchorIndex(visWithIdx, anchorKey, fallbackIdx=null){ for(let i=visWithIdx.length-1;i>=0;i--){ const candidate=_compressionMessageAnchorKey(visWithIdx[i].m); if(!candidate) continue; + const anchorTs=String(anchorKey.ts??''); + const candidateTs=String(candidate.ts??''); if( candidate.role===String(anchorKey.role||'') && - String(candidate.ts??'')===String(anchorKey.ts??'') && + (!anchorTs||!candidateTs||candidateTs===anchorTs) && String(candidate.text||'')===String(anchorKey.text||'') && Number(candidate.attachments||0)===Number(anchorKey.attachments||0) ){ @@ -4484,6 +4486,24 @@ function _compressionAnchorIndex(visWithIdx, anchorKey, fallbackIdx=null){ } return typeof fallbackIdx==='number' ? fallbackIdx : null; } +function _latestCompressionReferenceMessage(messages, summaryText=''){ + if(!Array.isArray(messages)||!messages.length) return {message:null, rawIdx:-1}; + const summaryNorm=String(summaryText||'').replace(/\s+/g,' ').trim(); + for(let i=messages.length-1;i>=0;i--){ + const m=messages[i]; + if(!_isContextCompactionMessage(m)) continue; + if(!summaryNorm) return {message:m, rawIdx:i}; + let content=''; + try{ + content=String(msgContent(m)||''); + }catch(_){ + content=String((m&&m.content)||''); + } + const contentNorm=content.replace(/\s+/g,' ').trim(); + if(contentNorm.includes(summaryNorm)) return {message:m, rawIdx:i}; + } + return {message:null, rawIdx:-1}; +} function _compressionReferenceCardHtml(text, open=false){ const preview=text.split(/\n+/).filter(Boolean).slice(0,2).join(' '); return ` @@ -4886,7 +4906,10 @@ function renderMessages(options){ $('emptyState').style.display=(vis.length||preservedCompressionTaskMessages.length)?'none':''; inner.innerHTML=''; const compressionNode=compressionState?_compressionCardsNode(compressionState):null; - const referenceMessage=S.messages.find(m=>_isContextCompactionMessage(m)); + const {message:referenceMessage, rawIdx:referenceMessageRawIdx}=_latestCompressionReferenceMessage( + S.messages, + sessionCompressionSummary + ); const referenceText=referenceMessage ? msgContent(referenceMessage)||String(referenceMessage.content||'') : sessionCompressionSummary; @@ -4937,13 +4960,19 @@ function renderMessages(options){ break; } } - const insertionAnchor=_compressionAnchorIndex( - renderVisWithIdx, + const insertionAnchorFull=_compressionAnchorIndex( + visWithIdx, compressionState ? compressionState.anchorMessageKey : sessionCompressionAnchorKey, compressionState ? (typeof compressionState.anchorVisibleIdx==='number' ? compressionState.anchorVisibleIdx : compressionState.anchorRawIdx) : sessionCompressionAnchor ); + let insertionAnchor=null; + if(typeof insertionAnchorFull==='number'){ + if(insertionAnchorFull=0) _insertCompressionLikeNodeByRawIdx(referenceNode, referenceMessageRawIdx); + else _insertCompressionLikeNode(referenceNode); _insertCompressionLikeNode(preservedOnlyNode, preservedOnlyAnchor); _insertCompressionLikeNode(handoffState?_handoffCardsNode(handoffState):null, renderVisWithIdx.length?renderVisWithIdx.length-1:null); for(const entry of handoffSummaryStates){ @@ -6383,8 +6413,12 @@ function appendThinking(text=''){ if(anchor) anchor.insertAdjacentElement('afterend', row); else blocks.appendChild(row); } - row.className=(text&&String(text).trim())?'assistant-segment thinking-card-row':'assistant-segment'; - row.innerHTML=_thinkingMarkup(text); + const clean=_sanitizeThinkingDisplayText(text); + const hasClean=!!String(clean||'').trim(); + row.className=hasClean?'assistant-segment thinking-card-row':'assistant-segment'; + const pre=row.querySelector('.thinking-card-body pre'); + if(pre&&hasClean) pre.textContent=String(clean).trim(); + else row.innerHTML=_thinkingMarkup(text); scrollIfPinned(); // Auto-scroll the thinking card body to bottom if the user is watching // (scroll pinned). If the user scrolled up to read history, leave it alone. @@ -6413,7 +6447,11 @@ function appendThinking(text=''){ row.setAttribute('data-thinking-active','1'); body.insertBefore(row, body.firstChild); } - row.innerHTML=_thinkingMarkup(text); + const clean=_sanitizeThinkingDisplayText(text); + const hasClean=!!String(clean||'').trim(); + const pre=row.querySelector('.thinking-card-body pre'); + if(pre&&hasClean) pre.textContent=String(clean).trim(); + else row.innerHTML=_thinkingMarkup(text); _syncToolCallGroupSummary(group); scrollIfPinned(); if(_scrollPinned){ diff --git a/tests/conftest.py b/tests/conftest.py index 8b993538fef..6d4e7ecc5a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,8 +2,8 @@ Shared pytest fixtures for webui-mvp tests. TEST ISOLATION: - Tests run against a SEPARATE server instance on port 8788 with a - completely separate state directory. Production data is never touched. + Tests run against a SEPARATE server instance on an auto-derived test port + with a completely separate state directory. Production data is never touched. The test state dir is wiped before each full test run and again on teardown. PATH DISCOVERY: @@ -32,7 +32,7 @@ # ── Test server config ──────────────────────────────────────────────────── # Port and state dir auto-derive from the repo path when no env var is set, -# giving every worktree its own isolated port (8800-8899) and state directory. +# giving every worktree its own isolated port (20000-29999) and state directory. # Override with HERMES_WEBUI_TEST_PORT / HERMES_WEBUI_TEST_STATE_DIR to pin. def _auto_test_port(repo_root) -> int: diff --git a/tests/test_1062_busy_input_modes.py b/tests/test_1062_busy_input_modes.py index bb7514d29f3..47ede9ab426 100644 --- a/tests/test_1062_busy_input_modes.py +++ b/tests/test_1062_busy_input_modes.py @@ -106,7 +106,7 @@ def test_cmd_steer_delegates_to_try_steer(self): # The shared helper must contain the fallback path helper_idx = COMMANDS_JS.find("async function _trySteer(") assert helper_idx >= 0, "_trySteer helper must exist" - helper_body = COMMANDS_JS[helper_idx:helper_idx + 1500] + helper_body = COMMANDS_JS[helper_idx:helper_idx + 2000] assert "queueSessionMessage" in helper_body assert "cancelStream" in helper_body # Toast should differ from interrupt to signal it's the steer path @@ -139,7 +139,7 @@ def test_slash_commands_clear_pending_files(self): # cmdSteer delegates to _trySteer; that helper clears pendingFiles idx_try = COMMANDS_JS.find("function _trySteer(") assert idx_try >= 0, "_trySteer not found" - try_body = COMMANDS_JS[idx_try:idx_try + 1200] + try_body = COMMANDS_JS[idx_try:idx_try + 1600] assert "S.pendingFiles=[]" in try_body, ( "_trySteer must clear S.pendingFiles in its fallback path — " "without this, files are lost on steer→interrupt fallback" @@ -268,7 +268,7 @@ def test_send_reads_busy_input_mode(self): def test_send_calls_cancel_stream_on_interrupt(self): send_idx = MESSAGES_JS.find("async function send(") - send_body = MESSAGES_JS[send_idx:send_idx + 3000] + send_body = MESSAGES_JS[send_idx:send_idx + 5000] # The interrupt branch must call cancelStream assert "cancelStream" in send_body # And queue before cancel (otherwise the drain has nothing to pick up) diff --git a/tests/test_auto_compression_card.py b/tests/test_auto_compression_card.py index 25571b2611f..f27760ec983 100644 --- a/tests/test_auto_compression_card.py +++ b/tests/test_auto_compression_card.py @@ -87,6 +87,28 @@ def test_auto_compression_sse_keeps_inactive_and_malformed_paths_safe(): assert guard in block assert block.index(guard) < block.index("setCompressionUi") assert "try{ d=JSON.parse(e.data||'{}')||{}; }catch(_){ d={}; }" in block + assert "if(d.session_id&&d.session_id!==activeSid) return;" in block + + +def test_auto_compression_done_sse_refreshes_context_indicator_usage(): + block = _compressed_listener_block() + + assert "if(d.usage&&typeof _syncCtxIndicator==='function')" in block + assert "S.lastUsage={...(S.lastUsage||{}),...d.usage};" in block + assert "_syncCtxIndicator(S.lastUsage);" in block + assert block.index("_syncCtxIndicator(S.lastUsage);") < block.index("setCompressionUi") + + +def test_auto_compression_done_payload_includes_live_usage_snapshot(): + src = _read("api/streaming.py") + start = src.find("put('compressed'") + assert start != -1, "compressed SSE payload not found" + end = src.find("})", start) + assert end != -1, "compressed SSE payload end not found" + block = src[start:end] + + assert "'session_id': s.session_id" in block + assert "'usage': _live_usage_snapshot()" in block def test_auto_compression_card_reuses_compression_card_renderer(): @@ -217,6 +239,66 @@ def test_context_anchor_reference_uses_session_summary_fallback(): assert "!!referenceText && (sessionCompressionAnchor!==null || sessionCompressionAnchorKey || sessionCompressionSummary)" in src +def test_compression_anchor_matching_tolerates_legacy_missing_timestamp(): + src = _read("static/ui.js") + start = src.find("function _compressionAnchorIndex") + assert start != -1, "compression anchor matcher not found" + end = src.find("function _compressionReferenceCardHtml", start) + assert end != -1, "compression reference renderer not found after anchor matcher" + helper = src[start:end] + + assert "const anchorTs=String(anchorKey.ts??'');" in helper + assert "const candidateTs=String(candidate.ts??'');" in helper + assert "(!anchorTs||!candidateTs||candidateTs===anchorTs)" in helper + + +def test_compression_anchor_index_is_translated_into_render_window(): + src = _read("static/ui.js") + start = src.find("const insertionAnchorFull=_compressionAnchorIndex") + assert start != -1, "full compression anchor lookup not found" + end = src.find("let _prevSepKey=null", start) + assert end != -1, "message render loop marker not found after anchor lookup" + block = src[start:end] + + assert "_compressionAnchorIndex(\n visWithIdx," in block + assert "insertionAnchorFull=0) _insertCompressionLikeNodeByRawIdx(referenceNode, referenceMessageRawIdx);" in src + assert "else _insertCompressionLikeNode(referenceNode);" in src + + +def test_reference_message_selection_prefers_latest_matching_marker(): + src = _read("static/ui.js") + start = src.find("function _latestCompressionReferenceMessage") + assert start != -1, "compression reference selection helper not found" + end = src.find("function _compressionReferenceCardHtml", start) + assert end != -1, "compression reference renderer not found after selection helper" + helper = src[start:end] + + assert "for(let i=messages.length-1;i>=0;i--)" in helper + assert "if(!summaryNorm) return {message:m, rawIdx:i};" in helper + assert "if(contentNorm.includes(summaryNorm)) return {message:m, rawIdx:i};" in helper + + +def test_reference_message_falls_back_to_current_summary_when_only_stale_markers_exist(): + src = _read("static/ui.js") + start = src.find("function _latestCompressionReferenceMessage") + assert start != -1, "compression reference selection helper not found" + end = src.find("function _compressionReferenceCardHtml", start) + assert end != -1, "compression reference renderer not found after selection helper" + helper = src[start:end] + + assert "const summaryNorm=String(summaryText||'').replace(/\\s+/g,' ').trim();" in helper + assert "return {message:null, rawIdx:-1};" in helper + + def test_preserved_task_list_attaches_once_per_render(): src = _read("static/ui.js") diff --git a/tests/test_compress_status_404_fix.py b/tests/test_compress_status_404_fix.py new file mode 100644 index 00000000000..d577f51c2f4 --- /dev/null +++ b/tests/test_compress_status_404_fix.py @@ -0,0 +1,139 @@ +""" +Regression test: /api/session/compress/status must not return 404. + +Bug: switching sessions triggered resumeManualCompressionForSession(), +which called GET /api/session/compress/status. The route handler returned +None (from j()) instead of True, causing do_GET's fallback to emit +{"error":"not found"} 404 in edge cases. The frontend then showed +"Compression failed: not found" toast on every session switch. + +Fix (two-part): + 1. Backend: handle_get now returns True after _handle_session_compress_status + 2. Frontend: resumeManualCompressionForSession catches 404 silently +""" + +import io +import json +from pathlib import Path + +from api.routes import _handle_session_compress_status, handle_get +from tests._pytest_port import BASE + + +# --------------------------------------------------------------------------- +# Reuse the _FakeHandler pattern from test_sprint46 +# --------------------------------------------------------------------------- +class _FakeHandler: + def __init__(self): + self.wfile = io.BytesIO() + self.status = None + self.sent_headers = {} + + def send_response(self, status): + self.status = status + + def send_header(self, key, value): + self.sent_headers[key] = value + + def end_headers(self): + pass + + def payload(self): + return json.loads(self.wfile.getvalue().decode("utf-8")) + + +# ======== Backend tests ======== + + +def test_compress_status_returns_200_idle_for_unknown_session(): + """The idle case (no active compression) must return 200, not 404.""" + handler = _FakeHandler() + result = _handle_session_compress_status(handler, "nonexistent_session_xyz") + body = handler.payload() + + assert handler.status == 200 + assert body["ok"] is True + assert body["status"] == "idle" + + +def test_compress_status_returns_200_idle_for_empty_session_id(): + """Empty session_id should return 400 (bad), not 404.""" + handler = _FakeHandler() + _handle_session_compress_status(handler, "") + assert handler.status == 400 + + +def test_handle_get_returns_true_for_compress_status(): + """handle_get must return True (not None/False) for compress/status. + + This is the core fix: previously it returned None (from j()), which + only worked because 'None is False' is False. But in edge cases + (stale process state, exception during response write) the fallback + could produce a 404. Returning True is defensive. + """ + from urllib.parse import urlparse + + handler = _FakeHandler() + parsed = urlparse("/api/session/compress/status?session_id=test_resume_sid") + result = handle_get(handler, parsed) + + assert result is True, f"handle_get returned {result!r}, expected True" + assert handler.status == 200 + + +def test_handle_get_returns_true_for_compress_status_no_sid(): + """Even with missing session_id, handle_get returns True (400 handled internally).""" + from urllib.parse import urlparse + + handler = _FakeHandler() + parsed = urlparse("/api/session/compress/status") + result = handle_get(handler, parsed) + + assert result is True + # _handle_session_compress_status should return 400 for empty sid + assert handler.status == 400 + + +# ======== Frontend static tests ======== + + +def _read_commands_js(): + with open( + Path(__file__).resolve().parents[1] / "static" / "commands.js", + encoding="utf-8", + ) as f: + return f.read() + + +def test_frontend_resume_404_silent(): + """resumeManualCompressionForSession must silently return on 404/5xx. + + The catch block should check for 404 and 5xx and return early, so + switching sessions never shows 'Compression failed' on transient errors. + """ + src = _read_commands_js() + + # Find the resumeManualCompressionForSession function + assert "async function resumeManualCompressionForSession" in src + + # The guard must be present in the catch block + assert "e.status===404" in src + assert "e.status>=500" in src + # Verify it's inside the catch block of resumeManualCompressionForSession + fn_start = src.index("async function resumeManualCompressionForSession") + # Find the catch block after this function + catch_idx = src.index("}catch(e){", fn_start) + guard_404 = src.index("e.status===404", fn_start) + guard_500 = src.index("e.status>=500", fn_start) + assert catch_idx < guard_404 < guard_500, "guards must be inside catch block" + + # The guard must return early (not just log) + line_with_guard = src[guard_404 : src.index("\n", guard_500) + 80] + assert "return" in line_with_guard, "guard must return early" + + +def test_frontend_compress_status_call_present(): + """Verify the compress/status API call is still in the frontend code.""" + src = _read_commands_js() + assert "/api/session/compress/status" in src + assert "resumeManualCompressionForSession" in src diff --git a/tests/test_docs_gitignore_policy.py b/tests/test_docs_gitignore_policy.py index a2729fae751..de0271db75f 100644 --- a/tests/test_docs_gitignore_policy.py +++ b/tests/test_docs_gitignore_policy.py @@ -20,6 +20,12 @@ def test_new_top_level_markdown_docs_are_trackable(): assert _git_check_ignore("docs/example-new-guide.md").returncode == 1 +def test_root_agents_entrypoint_is_trackable(): + """AGENTS.md is the shared repo entrypoint; local overrides stay ignored.""" + assert _git_check_ignore("AGENTS.md").returncode == 1 + assert _git_check_ignore("AGENTS.local.md").returncode == 0 + + def test_docs_scratch_files_remain_ignored(): """The broad docs/* ignore rule should still keep arbitrary scratch files out.""" assert _git_check_ignore("docs/local-scratch.tmp").returncode == 0 diff --git a/tests/test_security_redaction.py b/tests/test_security_redaction.py index 08ac49d6f82..7f908687cb2 100644 --- a/tests/test_security_redaction.py +++ b/tests/test_security_redaction.py @@ -107,6 +107,138 @@ def test_redact_value_list(): assert result[1]["content"] == "safe text" +def test_redact_text_skips_full_redactor_for_safe_text(monkeypatch): + """Large ordinary transcript text should not pay the full redactor pass.""" + import api.helpers as helpers + + calls = [] + monkeypatch.setattr(helpers, "_redact_fn_cached", lambda text: calls.append(text) or text) + + safe_text = "ordinary session transcript without credential markers\n" * 500 + assert helpers._redact_text(safe_text, _enabled=True) == safe_text + assert calls == [] + + +def test_redact_text_still_runs_full_redactor_for_sensitive_markers(monkeypatch): + """The cheap prefilter must preserve the hard redaction boundary.""" + import api.helpers as helpers + + calls = [] + + def fake_redactor(text): + calls.append(text) + return text.replace(_FAKE_SK_KEY, "sk-Tes...cdef") + + monkeypatch.setattr(helpers, "_redact_fn_cached", fake_redactor) + + result = helpers._redact_text(f"token={_FAKE_SK_KEY}", _enabled=True) + + assert _FAKE_SK_KEY not in result + assert calls == [f"token={_FAKE_SK_KEY}"] + + +@pytest.mark.parametrize("prefix,suffix", [ + ("sk-", "TestCredential1234567890"), + ("ghp_", "TestCredential1234567890"), + ("github_pat_", "TestCredential_1234567890"), + ("gho_", "TestCredential1234567890"), + ("ghu_", "TestCredential1234567890"), + ("ghs_", "TestCredential1234567890"), + ("ghr_", "TestCredential1234567890"), + ("xoxb-", "TestCredential1234567890"), + ("xoxa-", "TestCredential1234567890"), + ("xoxp-", "TestCredential1234567890"), + ("xoxr-", "TestCredential1234567890"), + ("xoxs-", "TestCredential1234567890"), + ("AIza", "TestCredential1234567890abcdefghi"), + ("pplx-", "TestCredential1234567890"), + ("fal_", "TestCredential1234567890"), + ("fc-", "TestCredential1234567890"), + ("bb_live_", "TestCredential1234567890"), + ("gAAAA", "TestCredential1234567890abcd"), + ("AKIA", "TESTCREDENTIAL12"), + ("sk_" + "live_", "TestCredential1234567890"), + ("sk_" + "test_", "TestCredential1234567890"), + ("rk_" + "live_", "TestCredential1234567890"), + ("SG.", "TestCredential1234567890"), + ("hf_", "TestCredential1234567890"), + ("r8_", "TestCredential1234567890"), + ("npm_", "TestCredential1234567890"), + ("pypi-", "TestCredential1234567890"), + ("dop_v1_", "TestCredential1234567890"), + ("doo_v1_", "TestCredential1234567890"), + ("am_", "TestCredential1234567890"), + ("sk_", "TestCredential1234567890"), + ("tvly-", "TestCredential1234567890"), + ("exa_", "TestCredential1234567890"), + ("gsk_", "TestCredential1234567890"), + ("syt_", "TestCredential1234567890"), + ("retaindb_", "TestCredential1234567890"), + ("hsk-", "TestCredential1234567890"), + ("mem0_", "TestCredential1234567890"), + ("brv_", "TestCredential1234567890"), +]) +def test_redact_text_prefilter_covers_known_prefixed_credentials(prefix, suffix): + """Every known prefix must still reach the hard redactor.""" + import api.helpers as helpers + + token = prefix + suffix + result = helpers._redact_text(f"credential={token}", _enabled=True) + + assert token not in result + + +@pytest.mark.parametrize("text", [ + # OAuth callback URL with `code=` query param + "https://example.com/callback?code=AUTH_OPAQUE_VALUE", + # URL userinfo (user:password embedded in scheme://) + "https://admin:supersecretpassword@api.example.com/v1", + # Signed-URL sensitive query param + "https://cdn.example.com/file.zip?signature=ABCDEFGHIJKL", + # Session-token query param + "https://example.com/dashboard?session=xyzABC999DEF", + # WebSocket URL with token query param + "wss://example.com/ws?token=jwt_ABCDEFGHIJ", + # FTP userinfo + "ftp://user:pwd@files.example.com/path", +]) +def test_redact_text_prefilter_routes_url_containing_strings_to_hard_redactor(text): + """Stage-348 Opus follow-up to PR #2171: the credential prefilter must + catch URL userinfo and sensitive query params so they still reach the + hard agent redactor instead of bypassing it. + + Pre-fix, the prefilter only listed specific DB scheme prefixes + (postgres://, mysql://, etc.) and a closed set of form keys, so OAuth + callback URLs pasted into chat could pass through to the response + verbatim. The fix adds the generic "://" marker so http(s)/ws(s)/ftp + URLs always route to the hard redactor. + + We test the *prefilter routing decision* here — `_might_contain_sensitive_text` + must return True for any URL-shaped string — rather than asserting on the + specific output of the agent redactor (which varies between hermes-agent + versions and CI vs local installs). + """ + import api.helpers as helpers + + assert helpers._might_contain_sensitive_text(text) is True, ( + f"URL-shaped string {text!r} should route to hard redactor but the " + f"prefilter rejected it. Pre-fix this allowed OAuth callback URLs, " + f"URL userinfo, and signed-URL query params to bypass redaction." + ) + + +def test_redact_text_prefilter_admits_plain_text_without_url_or_credentials(): + """Stage-348 follow-up companion: plain text with no URL or credential + marker still bypasses the hard redactor (the prefilter's whole purpose + is to skip the expensive pass when no markers are present).""" + import api.helpers as helpers + + assert helpers._might_contain_sensitive_text("Hi how are you today?") is False + assert helpers._might_contain_sensitive_text("The user said 'hello'") is False + assert helpers._might_contain_sensitive_text("") is False + assert helpers._might_contain_sensitive_text(None) is False # type: ignore[arg-type] + + def test_redact_value_works_with_legacy_agent_redact_signature(monkeypatch): """_redact_text must tolerate older redact_sensitive_text(text) signatures.""" fake_agent = types.ModuleType("agent") diff --git a/tests/test_session_tail_payload.py b/tests/test_session_tail_payload.py new file mode 100644 index 00000000000..236551e66ec --- /dev/null +++ b/tests/test_session_tail_payload.py @@ -0,0 +1,92 @@ +from types import SimpleNamespace +from unittest.mock import patch +from urllib.parse import urlparse + + +class _FakeSession: + def __init__(self, messages): + self.session_id = "tail_payload_001" + self.title = "Tail payload" + self.workspace = "/tmp" + self.model = "gpt-test" + self.model_provider = None + self.messages = messages + self.tool_calls = [ + {"name": "old-tool", "snippet": "historical snippet", "assistant_msg_idx": 0} + ] + self.input_tokens = 0 + self.output_tokens = 0 + self.estimated_cost = 0 + self.context_length = 1 + self.threshold_tokens = 0 + self.last_prompt_tokens = 0 + self.active_stream_id = None + self.pending_user_message = None + self.pending_attachments = [] + self.pending_started_at = None + self.composer_draft = {} + + def compact(self): + return { + "session_id": self.session_id, + "title": self.title, + "workspace": self.workspace, + "model": self.model, + "model_provider": self.model_provider, + "message_count": len(self.messages), + "context_length": self.context_length, + "threshold_tokens": self.threshold_tokens, + "last_prompt_tokens": self.last_prompt_tokens, + "active_stream_id": self.active_stream_id, + "pending_user_message": self.pending_user_message, + "composer_draft": self.composer_draft, + } + + +def _invoke(session): + import api.routes as routes + + captured = {} + + def fake_j(_handler, data, status=200, extra_headers=None): + captured["data"] = data + captured["status"] = status + return data + + parsed = urlparse("/api/session?session_id=tail_payload_001&messages=1&resolve_model=0&msg_limit=1") + with patch("api.routes.get_session", return_value=session), \ + patch("api.routes._clear_stale_stream_state", return_value=False), \ + patch("api.routes._lookup_cli_session_metadata", return_value={}), \ + patch("api.routes.redact_session_data", side_effect=lambda raw: raw), \ + patch("api.routes.j", side_effect=fake_j): + routes.handle_get(SimpleNamespace(), parsed) + return captured["data"]["session"] + + +def test_tail_window_omits_historical_tool_calls_when_messages_have_tool_metadata(): + session = _FakeSession([ + {"role": "user", "content": "older"}, + { + "role": "assistant", + "content": "visible", + "tool_calls": [{"id": "call_1", "function": {"name": "tool", "arguments": "{}"}}], + }, + ]) + + payload = _invoke(session) + + assert payload["messages"] == [session.messages[-1]] + assert payload["tool_calls"] == [] + assert payload["_messages_truncated"] is True + + +def test_tail_window_keeps_session_tool_calls_for_legacy_messages_without_metadata(): + session = _FakeSession([ + {"role": "user", "content": "older"}, + {"role": "assistant", "content": "visible legacy message"}, + ]) + + payload = _invoke(session) + + assert payload["messages"] == [session.messages[-1]] + assert payload["tool_calls"] == session.tool_calls