fix(bot-mode): service DMs use the active Hermes installation (salvage #100673)

evals/botmode-dm-matrix/README.md

@@ -0,0 +1,52 @@
+# Bot Mode native delivery matrix
+
+Production Electron/preload/backend/tool execution; loopback scripted inference only.
+Scratch HOME, HERMES_HOME, Python facade and CLI shims; no user profiles or credentials.
+Run from the repository root after installing root npm dependencies and building Desktop:
+
+```sh
+cp evals/botmode-dm-matrix/probe-dm-matrix.spec.ts apps/desktop/e2e/
+git apply --unidiff-zero evals/botmode-dm-matrix/mock-trigger.patch
+(cd apps/desktop && npm run build)
+# Set DISPLAY, current XAUTHORITY, XDG_RUNTIME_DIR and VIRTUAL_ENV first.
+(cd apps/desktop && HERMES_DESKTOP_CDP_PORT=off BOT_DM_SERVICE_PATH=1 \
+  BOT_DM_EVIDENCE=/tmp/botmode-dm-matrix-proof \
+  npx playwright test e2e/probe-dm-matrix.spec.ts --reporter=list)
+git apply --unidiff-zero -R evals/botmode-dm-matrix/mock-trigger.patch
+rm apps/desktop/e2e/probe-dm-matrix.spec.ts
+```
+
+`BOT_DM_SERVICE_PATH=1` creates a disposable Python venv facade with an adjacent
+worktree-pinned `hermes`; PATH contains a different inert `hermes` that exits 2
+and records every invocation. This simulates the reported stale-launcher service
+shape without modifying the shared interpreter or the user's installed launcher.
+Omit that flag only on the baseline: a PATH shim then pins all children to this tree.
+
+## Receipts
+
+Base: `40f2702b22a343cbc95647efd0bdffe8e0b3d9e9`.
+Normal PATH: two native cases passed. Default canonical Desktop owner remained
+unchanged while default -> quiet Beta CLI -> unowned Gamma CLI executed, Beta
+received its own completion and default received the Beta completion. Named
+persisted Bot Chat IDs remained `matrix-beta` and `matrix-gamma`; both used the
+scratch OS HOME as cwd, not the profile state directory. Gamma opened in Desktop
+and rendered its attributed input. This refutes #105323's `--in ~` ownership premise.
+
+Incoming live-owner case also passed without reload: wait for the attributed
+sender card to mount, then expand it. A rejected earlier probe expanded before
+asynchronous reconciliation had mounted the card; its final DOM already contained
+`Message from beta` and `show message`. Reload was never required.
+
+Stale PATH on base: real default message_agent acknowledged `sent`, wrong launcher
+recorded `-p beta chat --in ~ -c Bot Chat --create-if-missing -Q --query-file ...`,
+and Beta had zero input rows. With #100673 cherry-picked: same matrix, two passed;
+both nested children used the adjacent worktree entrypoint, no wrong-path marker.
+
+Artifacts retained separately under `/tmp/botmode-dm-matrix/{verified,service-base,service-fixed}`:
+SQLite snapshots, active lease snapshots, scoped process checkpoints, prompts,
+DOM text and screenshots. The `.live.json` ticket path is only for live-owner
+admission; unowned CLI fallback does not promise that receipt format.
+
+This establishes Linux native Electron behavior, not native Windows parity,
+real-model reasoning quality, or the separate 420-second tool-timeout report.
+No native Windows host was discovered in the available host inventory.

evals/botmode-dm-matrix/mock-trigger.patch

@@ -0,0 +1,20 @@
+diff --git a/tests-js/scripts/mock-server.ts b/tests-js/scripts/mock-server.ts
+index 6dffd2cf0f388..09c6de07d81b2 100644
+--- a/tests-js/scripts/mock-server.ts
++++ b/tests-js/scripts/mock-server.ts
+@@ -502,0 +503,15 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
++          const dmMatch = userText.match(/E2E_DM\(([^)]+)\)\[([\s\S]*)\]/)
++          const lastUserIndex = messages.lastIndexOf(lastUserMsg)
++          if (dmMatch && !messages.slice(lastUserIndex + 1).some(m => m?.role === 'tool')) {
++            const turn: ScriptedTurn = {
++              text: '',
++              toolCalls: [{ name: 'message_agent', args: { target: dmMatch[1], message: dmMatch[2] } }],
++            }
++            if (stream) {
++              streamScriptedTurn(res, model, turn)
++            } else {
++              nonStreamingScriptedTurn(res, model, turn)
++            }
++            return
++          }
++

evals/botmode-dm-matrix/probe-dm-matrix.spec.ts

@@ -0,0 +1,170 @@
+import { execFileSync, spawn } from 'node:child_process'
+import fs from 'node:fs'
+import path from 'node:path'
+import { buildAppEnv, createSandbox, launchDesktop, waitForAppReady, writeEnvFile, writeMockProviderConfig, type MockBackendFixture } from './fixtures'
+import { MOCK_REPLY, startMockServer } from '../../../tests-js/scripts/mock-server'
+import { expect, test } from './test'
+
+const repo = path.resolve(import.meta.dirname, '../../..')
+let python = path.join(process.env.VIRTUAL_ENV || path.join(repo, '.venv'), 'bin', 'python')
+let fixture: MockBackendFixture
+let env: Record<string, string>
+const evidence = process.env.BOT_DM_EVIDENCE || '/tmp/botmode-dm-matrix/native'
+
+test.beforeAll(async () => {
+  fs.mkdirSync(evidence, { recursive: true })
+  const sandbox = createSandbox('dm-delivery')
+  const mock = await startMockServer({ holdFirstCompletionContaining: 'Message from 🤖 beta (@beta): matrix-unowned-sentinel' })
+  for (const name of ['default', 'alpha', 'beta', 'gamma']) {
+    const home = name === 'default' ? sandbox.hermesHome : path.join(sandbox.hermesHome, 'profiles', name)
+    fs.mkdirSync(home, { recursive: true })
+    writeMockProviderConfig(home, mock.url)
+    writeEnvFile(home)
+    fs.writeFileSync(path.join(home, 'SOUL.md'), `# ${name}\nA Bot Mode teammate.\n`)
+    fs.writeFileSync(path.join(home, 'profile.yaml'), 'name: ' + name + '\nui_meta:\n  hermes-bots: {}\n')
+  }
+  if (process.env.BOT_DM_SERVICE_PATH === '1') {
+    const sourceVenv = path.dirname(path.dirname(python))
+    const runtime = path.join(sandbox.root, 'runtime')
+    fs.mkdirSync(path.join(runtime, 'bin'), { recursive: true })
+    fs.copyFileSync(path.join(sourceVenv, 'pyvenv.cfg'), path.join(runtime, 'pyvenv.cfg'))
+    fs.symlinkSync(path.join(sourceVenv, 'lib'), path.join(runtime, 'lib'))
+    fs.symlinkSync(python, path.join(runtime, 'bin', 'python'))
+    python = path.join(runtime, 'bin', 'python')
+    fs.writeFileSync(path.join(runtime, 'bin', 'hermes'), `#!/bin/sh\ncd ${repo}\nexec ${python} -m hermes_cli.main "$@"\n`, { mode: 0o755 })
+  }
+  const bin = path.join(sandbox.root, 'bin')
+  fs.mkdirSync(bin)
+  fs.writeFileSync(path.join(bin, 'hermes'), `#!/bin/sh\ncd ${repo}\nexec ${python} -m hermes_cli.main "$@"\n`, { mode: 0o755 })
+  if (process.env.BOT_DM_SERVICE_PATH === '1') {
+    fs.writeFileSync(path.join(bin, 'hermes'), `#!/bin/sh\nprintf 'WRONG_PATH_HERMES invoked: %s\\n' "$*" >> ${path.join(evidence, 'wrong-path.log')}\nprintf 'old launcher rejects --query-file\\n' >&2\nexit 2\n`, { mode: 0o755 })
+  }
+  env = buildAppEnv(sandbox, { HOME: sandbox.root, HERMES_DESKTOP_PYTHON: python,
+    HERMES_DESKTOP_HERMES: path.join(bin, 'hermes'), PATH: `${bin}:${process.env.PATH}`,
+    PYTHONPATH: repo, HERMES_SINGLE_QUERY_LINGER_SECONDS: '30' })
+  for (const name of ['alpha', 'beta', 'gamma']) {
+    const h = path.join(sandbox.hermesHome, 'profiles', name)
+    execFileSync(python, ['-c', 'import sys; from pathlib import Path; from hermes_state import SessionDB; d=SessionDB(db_path=Path(sys.argv[1])/"state.db"); d.create_session("matrix-"+sys.argv[2],"cli",cwd=sys.argv[3]); d.set_session_title("matrix-"+sys.argv[2],"Bot Chat"); d.close()', h, name, sandbox.root], { env, cwd: repo })
+  }
+  const { app, page } = await launchDesktop(env)
+  fixture = { app, page, sandbox, mock, mockUrl: mock.url, cleanup: async () => {
+    await app.close().catch(() => undefined)
+    await mock.close()
+  } }
+  console.log('SANDBOX', sandbox.root)
+  fs.writeFileSync(path.join(evidence, 'sandbox.txt'), sandbox.root)
+  await waitForAppReady(fixture, 120_000)
+})
+
+test.afterAll(async () => { await fixture?.cleanup() })
+
+async function openBot(name: string) {
+  const page = fixture.page
+  await page.getByRole('button', { name: 'Bots', exact: true }).or(page.getByRole('tab', { name: 'Bots', exact: true })).first().click()
+  const row = page.getByRole('button', { name: new RegExp(`^${name === 'default' ? 'hermes' : name}\\b`, 'i') }).filter({ visible: true }).first()
+  await expect(row).toBeVisible({ timeout: 30_000 })
+  await row.click()
+  const composer = page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first()
+  await expect(composer).toBeVisible({ timeout: 120_000 })
+  return composer
+}
+
+function dbMessages(name: string) {
+  const home = name === 'default' ? fixture.sandbox.hermesHome : path.join(fixture.sandbox.hermesHome, 'profiles', name)
+  return JSON.parse(execFileSync(python, ['-c', 'import sqlite3,json,sys; c=sqlite3.connect(sys.argv[1]); print(json.dumps(c.execute("select role,content from messages").fetchall()))', path.join(home, 'state.db')], { env, cwd: repo, encoding: 'utf8' })) as string[][]
+}
+
+
+function py(code: string, args: string[] = []) {
+  return JSON.parse(execFileSync(python, ['-c', code, ...args], { env, cwd: repo, encoding: 'utf8' }))
+}
+function owner(name: string) {
+  return py('import json,sys; from tools.bot_live_delivery import find_canonical_live_owner; print(json.dumps(find_canonical_live_owner(sys.argv[1])))', [home(name)])
+}
+function home(name: string) {
+  return name === 'default' ? fixture.sandbox.hermesHome : path.join(fixture.sandbox.hermesHome, 'profiles', name)
+}
+function snapshot(name: string) {
+  return py('import json,sqlite3,sys; from hermes_cli.active_sessions import active_session_registry_snapshot; c=sqlite3.connect(sys.argv[1]+"/state.db"); c.row_factory=sqlite3.Row; print(json.dumps(dict(sessions=[dict(x) for x in c.execute("select * from sessions")],messages=[dict(x) for x in c.execute("select * from messages")],owners=active_session_registry_snapshot(registry_home=sys.argv[1])),default=str))', [home(name)])
+}
+async function capture(label: string) {
+  fs.writeFileSync(path.join(evidence, `${label}.json`), JSON.stringify({ default: snapshot('default'), alpha: snapshot('alpha'), beta: snapshot('beta'), gamma: snapshot('gamma'), processes: Object.fromEntries(['default', 'alpha', 'beta', 'gamma'].map(n => { const file = path.join(home(n), 'processes.json'); return [n, fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null] })), prompts: fixture.mock.receivedPrompts, dom: await fixture.page.locator('body').innerText() }, null, 2))
+  await fixture.page.screenshot({ path: path.join(evidence, `${label}.png`) })
+}
+async function unfold() {
+  for (const toggle of await fixture.page.getByText('show message', { exact: true }).all()) {
+    if (await toggle.isVisible()) await toggle.click()
+  }
+}
+
+test('default live owner does not capture named unowned nested quiet CLI deliveries', async () => {
+  test.setTimeout(300_000)
+  const page = fixture.page
+  const composer = await openBot('default')
+  await expect(page.getByText('Say something to get started.').filter({ visible: true })).toBeVisible({ timeout: 120_000 })
+  await composer.fill('initialize default owner')
+  await page.keyboard.press('Enter')
+  await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 })
+  const defaultOwner = owner('default')
+  expect(defaultOwner).not.toBeNull()
+  expect(owner('beta')).toBeNull()
+  expect(owner('gamma')).toBeNull()
+  // A Desktop turn executes the real message_agent, spawning quiet CLI B,
+  // which executes the real message_agent again into unowned C.
+  await composer.fill('E2E_DM(beta)[E2E_DM(gamma)[matrix-unowned-sentinel]]')
+  await page.keyboard.press('Enter')
+  try {
+    await expect.poll(() => dbMessages('beta').filter(([r,t]) => r === 'user' && t.includes('matrix-unowned-sentinel')).length, { timeout: 35_000 }).toBe(1)
+    await fixture.mock.waitForHeldCompletion()
+    await expect.poll(() => dbMessages('beta').filter(([r]) => r === 'assistant').length, { timeout: 60_000 }).toBeGreaterThanOrEqual(2)
+    expect(owner('default')).toEqual(defaultOwner)
+    await capture('nested-held')
+    const held = JSON.parse(fs.readFileSync(path.join(evidence, 'nested-held.json'), 'utf8'))
+    const nestedProcess = held.processes.beta[0]
+    expect(nestedProcess.session_key).toBe('matrix-beta')
+    expect(nestedProcess.parent_session_id).toBe('matrix-beta')
+    expect(nestedProcess.owner_task_id).toBeTruthy()
+    expect(nestedProcess.owner_task_id).not.toBe(defaultOwner.session_id)
+    expect(held.beta.owners[0].surface).toBe('cli')
+    expect(held.gamma.owners[0].surface).toBe('cli')
+    expect(held.beta.owners[0].pid).not.toBe(held.gamma.owners[0].pid)
+    expect(held.beta.sessions.filter((s: any) => s.title === 'Bot Chat').map((s: any) => s.id)).toEqual(['matrix-beta'])
+    expect(held.gamma.sessions.filter((s: any) => s.title === 'Bot Chat').map((s: any) => s.id)).toEqual(['matrix-gamma'])
+    fixture.mock.releaseHeldStream()
+    await expect.poll(() => dbMessages('beta').filter(([r,t]) => r === 'user' && t.includes('completed')).length, { timeout: 100_000 }).toBeGreaterThan(0)
+    await expect.poll(() => dbMessages('default').filter(([r,t]) => r === 'user' && t.includes('completed')).length, { timeout: 100_000 }).toBeGreaterThan(0)
+    expect(dbMessages('gamma').filter(([r,t]) => r === 'user' && t.includes('matrix-unowned-sentinel'))).toHaveLength(1)
+    expect(dbMessages('default').filter(([r,t]) => r === 'user' && t.startsWith('Message from'))).toHaveLength(0)
+    expect(owner('default')).toEqual(defaultOwner)
+    await openBot('gamma')
+    await unfold()
+    await expect(page.getByText(/matrix-unowned-sentinel/).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 })
+    await capture('nested-complete')
+    expect(dbMessages('beta').filter(([r,t]) => r === 'user' && t.startsWith('[IMPORTANT: Background process') && t.includes(nestedProcess.session_id))).toHaveLength(1)
+    expect(dbMessages('gamma').filter(([r,t]) => r === 'user' && t.startsWith('[IMPORTANT: Background process'))).toHaveLength(0)
+    expect(fs.existsSync(path.join(evidence, 'wrong-path.log'))).toBe(false)
+  } finally { fixture.mock.releaseHeldStream(); await capture('nested-final') }
+})
+
+test('incoming live sender card is rendered before any reload', async () => {
+  test.setTimeout(240_000)
+  const page = fixture.page
+  const composer = await openBot('alpha')
+  await expect(page.getByText('Say something to get started.').filter({ visible: true })).toBeVisible({ timeout: 120_000 })
+  await composer.fill('initialize alpha owner')
+  await page.keyboard.press('Enter')
+  await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 })
+  expect(owner('alpha')).not.toBeNull()
+  const output = fs.openSync(path.join(evidence, 'live-sender.log'), 'w')
+  const child = spawn(python, ['-m', 'hermes_cli.main', '-p', 'beta', 'chat', '--in', '~', '-c', 'Bot Chat', '--create-if-missing', '-Q', '-q', 'E2E_DM(alpha)[matrix-live-sender-card]'], { cwd: repo, env, stdio: ['ignore', output, output] })
+  try {
+    await expect.poll(() => dbMessages('alpha').filter(([r,t]) => r === 'user' && t.includes('matrix-live-sender-card')).length, { timeout: 100_000 }).toBe(1)
+    await expect.poll(() => child.exitCode, { timeout: 100_000 }).toBe(0)
+    await capture('live-before-unfold')
+    await expect(page.getByText('Message from beta', { exact: true }).filter({ visible: true }).first()).toBeVisible({ timeout: 30_000 })
+    await expect(page.getByText('show message', { exact: true }).filter({ visible: true }).first()).toBeVisible({ timeout: 30_000 })
+    await unfold()
+    await capture('live-after-unfold')
+    await expect(page.getByText(/matrix-live-sender-card/).filter({ visible: true }).first()).toBeVisible({ timeout: 30_000 })
+  } finally { child.kill(); fs.closeSync(output); await capture('live-final') }
+})

tests/tools/test_bot_mode_dm.py

@@ -16,7 +16,7 @@
 
 import pytest
 
-from tools import bot_mode_dm, bot_mode_probe
+from tools import bot_mode_dm, bot_mode_probe, bot_relay
 
 
 @pytest.fixture(autouse=True)
@@ -234,6 +234,9 @@ def _runner_author(command):
 
 def test_local_delivery_command_and_ack(tmp_path, monkeypatch):
     calls = _capture_spawn(monkeypatch)
+    # These assertions target the -p/turn-args shape; pin the entrypoint resolution
+    # so the test stays hermetic across venvs that do/don't expose a sibling script.
+    monkeypatch.setattr(bot_relay, "_hermes_cli", lambda: "hermes")
     home = _managed_home(tmp_path, teammates=("researcher",))
     agent = _FakeAgent(home, title="Bot Chat")
 
@@ -295,6 +298,7 @@ def test_peer_delivery_command_pins_registry_profile_for_secondary_bots(
     tool-side roster (read from the machine-root config) validated the
     target."""
     calls = _capture_spawn(monkeypatch)
+    monkeypatch.setattr(bot_relay, "_hermes_cli", lambda: "hermes")
     home = _managed_home(tmp_path, peers=("spark",))
     # A reviewer-profile gateway context: the agent's session db lives under
     # that profile's home, so _agent_home() resolves there while the
@@ -316,6 +320,7 @@ def test_peer_delivery_command_pins_registry_profile_for_secondary_bots(
 
 def test_peer_delivery_command(tmp_path, monkeypatch):
     calls = _capture_spawn(monkeypatch)
+    monkeypatch.setattr(bot_relay, "_hermes_cli", lambda: "hermes")
     monkeypatch.setattr("socket.gethostname", lambda: "eri-mac.local")
     home = _managed_home(tmp_path, peers=("spark",))
     agent = _FakeAgent(home, title="Bot Chat")
@@ -341,6 +346,40 @@ def test_peer_delivery_command(tmp_path, monkeypatch):
     assert transport_argv == ["hermes", "-p", "default", "peer", "dm", "spark"]
 
 
+def test_delivery_pins_the_hermes_entrypoint_beside_this_interpreter(tmp_path, monkeypatch):
+    """A background delivery must not rely on PATH: the runner's service context
+    lacks the gateway's venv bin dir, so a bare ``hermes`` resolves to a system
+    install whose shebang picks the wrong interpreter and dies on import (#108628).
+    Both transports must invoke the entrypoint beside this interpreter instead."""
+    venv_bin = tmp_path / "venv" / ("Scripts" if sys.platform == "win32" else "bin")
+    venv_bin.mkdir(parents=True)
+    hermes_entry = venv_bin / ("hermes.exe" if sys.platform == "win32" else "hermes")
+    hermes_entry.write_text("#!/bin/sh\n", encoding="utf-8")
+    monkeypatch.setattr(sys, "executable", str(venv_bin / "python3"))
+
+    calls = _capture_spawn(monkeypatch)
+    home = _managed_home(tmp_path, teammates=("researcher",), peers=("spark",))
+    agent = _FakeAgent(home, title="Bot Chat")
+
+    result = json.loads(
+        bot_mode_dm.message_agent_tool(target="researcher", message="ping", agent=agent)
+    )
+    assert result["status"] == "sent"
+    mode, _dm_file, transport_argv = _runner_parts(calls[0]["command"])
+    assert mode == "query-file"
+    assert transport_argv[0] == str(hermes_entry)
+    assert transport_argv[1:] == ["-p", "researcher", "chat", "--in", "~", "-c", "Bot Chat",
+                                  "--create-if-missing", "-Q"]
+
+    result2 = json.loads(
+        bot_mode_dm.message_agent_tool(target="spark", message="ping", agent=agent)
+    )
+    assert result2["status"] == "sent"
+    mode, _dm_file, transport_argv = _runner_parts(calls[1]["command"])
+    assert mode == "stdin"
+    assert transport_argv == [str(hermes_entry), "-p", "default", "peer", "dm", "spark"]
+
+
 def test_peer_delivery_author_carries_the_sender_hostname_and_local_stays_bare(tmp_path, monkeypatch):
     """A peer dm crosses installs, so its author id is ``bot:<hostname>/<profile>``: the peer's own ``coder`` and a
     remote ``coder`` must not share one id. A teammate on this install still sees the bare ``bot:coder``."""