From 0d28624502a22c0af86c5ca5eae0a730efd2611c Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Tue, 12 May 2026 17:06:22 +0100 Subject: [PATCH 01/11] fix(mcp): pre-install MCP servers globally to skip npx cache contention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Zed and Claude Code spawn in parallel against the shared `_npx/` cache, npm's "reify mark retired" dance (rename node_modules/ → .-, reinstall, rename back) races and the MCP server's JSON-RPC `initialize` never returns. Spec-task containers then surface ERROR [project::context_server_store] chrome-devtools context server failed to start: Context server request timeout after the 180s `context_server_timeout` fires for chrome-devtools and github (drone-ci was unaffected because it was already globally installed at /usr/bin/drone-ci-mcp, so npm exec resolved via PATH and skipped the cache entirely). Manual reproduction with the exact same protocol version, env vars and shell wrapping Zed uses returns initialize in <2s, confirming the hang is in the npx install path, not the MCP server itself. Fix: - Dockerfile.ubuntu-helix: pin and globally install `chrome-devtools-mcp` and `@modelcontextprotocol/server-github` next to the existing global drone-ci-mcp install. - zed_config.go: invoke `/usr/bin/chrome-devtools-mcp` directly instead of `npx chrome-devtools-mcp@latest` so the cache is never touched. - simple_sample_projects.go: switch the Helix-in-Helix sample's GitHub MCP from `npx -y @modelcontextprotocol/server-github` to the global `mcp-server-github` binary for the same reason. Co-Authored-By: Claude Opus 4.7 --- Dockerfile.ubuntu-helix | 15 +++++++++++++-- api/pkg/external-agent/zed_config.go | 12 ++++++++++-- api/pkg/server/simple_sample_projects.go | 17 ++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/Dockerfile.ubuntu-helix b/Dockerfile.ubuntu-helix index e736420962..2488aba962 100644 --- a/Dockerfile.ubuntu-helix +++ b/Dockerfile.ubuntu-helix @@ -884,8 +884,19 @@ RUN mkdir -p /etc/skel/.config/google-chrome && \ echo '{"browser":{"has_seen_welcome_page":true},"distribution":{"skip_first_run_ui":true}}' \ > /etc/skel/.config/google-chrome/Default/Preferences -# Install chrome-devtools-mcp globally and pre-build fontconfig cache -RUN npm install -g chrome-devtools-mcp@latest && fc-cache -f -v +# Install MCP server packages globally so Zed and Claude Code can invoke them +# by direct binary path (e.g. /usr/bin/chrome-devtools-mcp) instead of +# `npx @latest`. Going through npx hits the shared _npx cache, and when +# Zed and Claude Code spawn in parallel against the same cache the install +# dance ("reify mark retired" — rename node_modules/ → tmp, reinstall, +# rename back) races and the JSON-RPC `initialize` never returns. Spec-task +# logs then show `chrome-devtools context server failed to start: Context +# server request timeout` (180s). Going via the global binary avoids npx +# entirely and removes the contention. +RUN npm install -g \ + chrome-devtools-mcp@0.25.0 \ + @modelcontextprotocol/server-github@2025.4.8 \ + && fc-cache -f -v # URL capture script for Claude Code OAuth login flow. # When BROWSER=/usr/local/bin/helix-capture-browser, `claude auth login` writes diff --git a/api/pkg/external-agent/zed_config.go b/api/pkg/external-agent/zed_config.go index 86453c23e2..8e4b901ae9 100644 --- a/api/pkg/external-agent/zed_config.go +++ b/api/pkg/external-agent/zed_config.go @@ -287,8 +287,17 @@ func GenerateZedMCPConfig( // console access, network analysis, and input automation. // Uses Puppeteer internally to control Chrome via CDP (Chrome DevTools Protocol). // See: https://developer.chrome.com/blog/chrome-devtools-mcp + // + // Invoke the globally-installed binary directly (Dockerfile.ubuntu-helix + // pins `chrome-devtools-mcp` via `npm install -g`). Going through + // `npx chrome-devtools-mcp@latest` instead causes npm's `_npx/` + // cache to do a "reify mark retired" rename dance every spawn; when Zed + // and Claude Code spawn in parallel the renames race and the JSON-RPC + // `initialize` never returns — Zed surfaces this as + // `chrome-devtools context server failed to start: Context server + // request timeout` (180s). config.ContextServers["chrome-devtools"] = ContextServerConfig{ - Command: "npx", + Command: "/usr/bin/chrome-devtools-mcp", // --viewport sets the rendered page size (Chrome window ends up viewport + ~80px // of decorations). 1280x800 sits at the canonical desktop-vs-mobile breakpoint // so sites still render in desktop mode, and the resulting Chrome window leaves @@ -298,7 +307,6 @@ func GenerateZedMCPConfig( // Disables navigator.webdriver, suppresses "Chrome is being controlled" infobar, // and prevents extension probing (e.g. LinkedIn bot detection). Args: []string{ - "chrome-devtools-mcp@latest", "--viewport", "1280x800", "--chrome-arg=--disable-blink-features=AutomationControlled", "--chrome-arg=--no-first-run", diff --git a/api/pkg/server/simple_sample_projects.go b/api/pkg/server/simple_sample_projects.go index d4d3c29498..b23ec03e76 100644 --- a/api/pkg/server/simple_sample_projects.go +++ b/api/pkg/server/simple_sample_projects.go @@ -678,11 +678,18 @@ This is IMPERATIVE - if you don't record and push the color, it cannot be cloned }, }, { - Name: "GitHub", - Description: "Interact with GitHub repositories, issues, pull requests, and more", - Transport: "stdio", - Command: "npx", - Args: []string{"-y", "@modelcontextprotocol/server-github"}, + Name: "GitHub", + Description: "Interact with GitHub repositories, issues, pull requests, and more", + Transport: "stdio", + // Globally installed in the desktop image (see Dockerfile.ubuntu-helix — + // `npm install -g @modelcontextprotocol/server-github`). Going through + // `npx -y @modelcontextprotocol/server-github` instead causes npm's + // _npx cache "reify mark retired" rename dance to race against the + // parallel chrome-devtools spawn from Zed/Claude Code, and the + // JSON-RPC `initialize` call hangs until the 180s context_server + // timeout fires. + Command: "mcp-server-github", + Args: []string{}, OAuthProvider: "github", // Reuse the GitHub OAuth connection from this sample project }, }, From cda23f95571a3a5a5ee3707b0ecea5f842112145 Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Tue, 12 May 2026 17:38:47 +0100 Subject: [PATCH 02/11] fix(mcp): use globally-installed binaries in frontend skill dialogs and docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same root cause as the previous commit: `npx -y ` for github and drone-ci spawns into the shared `_npx/` cache and races against the parallel chrome-devtools spawn from Zed/Claude Code, causing the JSON-RPC `initialize` to hang past the 180s timeout. The previous commit fixed the server-side defaults (zed_config.go hardcoded chrome-devtools entry, simple_sample_projects.go GitHub sample). But the actual config that lands in user projects comes from the frontend skill dialogs whenever a user enables a skill in Project Settings — and those were still hardcoded to `npx -y …`. Result: every project with the GitHub skill enabled keeps writing the broken config back into the project row, even on a freshly-fixed backend. Fix the same set of frontend hardcoded sites and the doc/example YAML: - `GitHubMcpSkill.tsx` (clicking "Enable" on the GitHub skill): command: 'mcp-server-github', args: []. - `AddLocalMcpSkillDialog.tsx` (the "Drone CI" example chip and the command-line placeholder): drone-ci-mcp instead of npx -y drone-ci-mcp. - `Skills.tsx`: tweak the "New Local MCP" skill description to reflect the global-binary pattern. - `examples/project.yaml`, `docs/helix-apply.md`: switch the GitHub MCP example from npx to the global binary. Existing user projects that already have the broken `npx -y …` config will continue to use it until the user re-saves the skill in Project Settings — that's user data and we can't migrate it from here. Co-Authored-By: Claude Opus 4.7 --- docs/helix-apply.md | 14 ++++++++++---- examples/project.yaml | 7 +++++-- .../src/components/app/AddLocalMcpSkillDialog.tsx | 11 ++++++++--- frontend/src/components/app/GitHubMcpSkill.tsx | 14 +++++++++++--- frontend/src/components/app/Skills.tsx | 2 +- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/helix-apply.md b/docs/helix-apply.md index 7bb815fa9d..30396c2e89 100644 --- a/docs/helix-apply.md +++ b/docs/helix-apply.md @@ -85,10 +85,13 @@ spec: web_search: true browser: false mcps: + # github MCP is pre-installed globally in the desktop image — invoke + # the binary directly. `npx -y @modelcontextprotocol/server-github` + # also works but races on the shared _npx cache when spawned in + # parallel with chrome-devtools, causing 180s initialize timeouts. - name: github transport: stdio - command: npx - args: ["-y", "@modelcontextprotocol/server-github"] + command: mcp-server-github env: GITHUB_TOKEN: "${GITHUB_TOKEN}" ``` @@ -234,10 +237,13 @@ spec: browser: false calculator: false mcps: + # github MCP is pre-installed globally in the desktop image — invoke + # the binary directly. `npx -y @modelcontextprotocol/server-github` + # also works but races on the shared _npx cache when spawned in + # parallel with chrome-devtools, causing 180s initialize timeouts. - name: github transport: stdio - command: npx - args: ["-y", "@modelcontextprotocol/server-github"] + command: mcp-server-github env: GITHUB_TOKEN: "${GITHUB_TOKEN}" knowledge: diff --git a/examples/project.yaml b/examples/project.yaml index e6846fe037..6ae94b5d05 100644 --- a/examples/project.yaml +++ b/examples/project.yaml @@ -65,9 +65,12 @@ spec: tools: web_search: true mcps: + # The github MCP server is pre-installed globally in the spec-task + # desktop image; invoke it by binary name. Using `npx -y` instead + # races on the shared npm cache when spawned in parallel with + # chrome-devtools and times out after 180s. - name: github transport: stdio - command: npx - args: ["-y", "@modelcontextprotocol/server-github"] + command: mcp-server-github env: GITHUB_TOKEN: "${GITHUB_TOKEN}" diff --git a/frontend/src/components/app/AddLocalMcpSkillDialog.tsx b/frontend/src/components/app/AddLocalMcpSkillDialog.tsx index 52f958ba4d..7042975f5e 100644 --- a/frontend/src/components/app/AddLocalMcpSkillDialog.tsx +++ b/frontend/src/components/app/AddLocalMcpSkillDialog.tsx @@ -305,14 +305,19 @@ const AddLocalMcpSkillDialog: React.FC = ({ }} onClick={() => { setName('Drone CI'); - setCommandLine('npx -y drone-ci-mcp'); + // drone-ci-mcp is globally installed in the desktop image + // (Dockerfile.ubuntu-helix) — invoke the binary directly + // rather than via `npx -y`. npx hits the shared _npx cache + // and races against parallel chrome-devtools spawns, causing + // 180s context_server initialize timeouts. + setCommandLine('drone-ci-mcp'); setEnv({ 'DRONE_SERVER_URL': 'https://drone.example.com', 'DRONE_ACCESS_TOKEN': 'your-drone-api-token', }); }} > - npx -y drone-ci-mcp + drone-ci-mcp = ({ onChange={(e) => setCommandLine(e.target.value)} margin="normal" required - placeholder="npx -y drone-ci-mcp" + placeholder="drone-ci-mcp" /> {/* Preview how command is parsed */} diff --git a/frontend/src/components/app/GitHubMcpSkill.tsx b/frontend/src/components/app/GitHubMcpSkill.tsx index afe4910bce..d0aada3df8 100644 --- a/frontend/src/components/app/GitHubMcpSkill.tsx +++ b/frontend/src/components/app/GitHubMcpSkill.tsx @@ -160,13 +160,21 @@ const GitHubMcpSkill: React.FC = ({ return; } - // Create the MCP skill object using official GitHub MCP server + // Create the MCP skill object using official GitHub MCP server. + // Invoke the globally-installed binary directly — Dockerfile.ubuntu-helix + // pins `@modelcontextprotocol/server-github` via `npm install -g`. + // Going through `npx -y @modelcontextprotocol/server-github` instead + // hits the shared `_npx/` cache, and when Zed and Claude Code + // spawn in parallel the npm "reify mark retired" rename dance races + // and the JSON-RPC `initialize` never returns — Zed surfaces this as + // `github context server failed to start: Context server request + // timeout` (180s). const mcpSkill: TypesAssistantMCP = { name: GITHUB_MCP_NAME, description: 'GitHub integration for issues, PRs, repos, and more', transport: 'stdio', - command: 'npx', - args: ['-y', '@modelcontextprotocol/server-github'], + command: 'mcp-server-github', + args: [], }; if (authMethod === 'oauth') { diff --git a/frontend/src/components/app/Skills.tsx b/frontend/src/components/app/Skills.tsx index 2c5e15d01b..b8c521d913 100644 --- a/frontend/src/components/app/Skills.tsx +++ b/frontend/src/components/app/Skills.tsx @@ -365,7 +365,7 @@ const CUSTOM_LOCAL_MCP_SKILL: ISkill = { id: 'new-local-mcp', icon: , name: 'New Local MCP', - description: 'Add a local MCP server that runs inside the dev container. Perfect for npx-based MCPs like drone-ci-mcp.', + description: 'Add a local MCP server that runs inside the dev container. Use the binary name (e.g. drone-ci-mcp) when the package is pre-installed in the desktop image.', type: SKILL_TYPE_LOCAL_MCP, categories: [SKILL_CATEGORY_CORE, SKILL_CATEGORY_LOCAL_MCP], skill: { From d158601dc6f067e877eb77066b65526070a2aa2d Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 09:05:58 +0100 Subject: [PATCH 03/11] fix(mcp): add per-spawn isolated npx shim for user-provided MCP servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds /usr/local/bin/npx shim that gives each invocation its own NPM_CONFIG_CACHE, avoiding the npm `_npx/` "reify mark retired" rename race when multiple npx invocations target the same package in parallel. Caveat: Zed prepends its OWN bundled `~/.local/share/zed/node/.../bin` to PATH before /usr/local/bin when spawning context_servers and the Claude ACP wrapper, so this shim is bypassed for Zed-launched MCPs. The shim still helps for: - User-typed `command: "npx"` MCPs spawned from a non-Zed shell context (e.g. helix CLI, terminal sessions inside the dev container). - Defense-in-depth if/when Zed's PATH ordering changes. A complete fix needs to also shim Zed's bundled npx after it gets downloaded — TODO follow-up. The deeper fix is to remove the parallel spawn entirely (helixml/zed duplicate Claude ACP spawn) so the cache is never contended in the first place. Co-Authored-By: Claude Opus 4.7 --- Dockerfile.ubuntu-helix | 10 +++++++- desktop/shared/helix-npx.sh | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 desktop/shared/helix-npx.sh diff --git a/Dockerfile.ubuntu-helix b/Dockerfile.ubuntu-helix index 2488aba962..7152b3af71 100644 --- a/Dockerfile.ubuntu-helix +++ b/Dockerfile.ubuntu-helix @@ -904,7 +904,15 @@ RUN npm install -g \ # for this file and opens the URL in the user's native browser. COPY desktop/shared/helix-capture-browser.sh /usr/local/bin/helix-capture-browser COPY desktop/shared/helix-claude-auth-wrapper.sh /usr/local/bin/helix-claude-auth-wrapper -RUN chmod +x /usr/local/bin/helix-capture-browser /usr/local/bin/helix-claude-auth-wrapper +# helix-npx — installed as /usr/local/bin/npx so it shadows the system +# /usr/bin/npx via PATH order. Gives each `npx ` invocation its own +# NPM_CONFIG_CACHE so parallel npx spawns (Zed + Claude both starting +# the same MCP at session start, or two parallel agent sessions hitting +# the same package) don't race in npm's `_npx/` rename dance. +# Zed's own ACP-wrapper bootstrapping calls npm via absolute path so it +# bypasses this shim. +COPY desktop/shared/helix-npx.sh /usr/local/bin/npx +RUN chmod +x /usr/local/bin/helix-capture-browser /usr/local/bin/helix-claude-auth-wrapper /usr/local/bin/npx # Install drone-ci-mcp (Helix's Drone CI MCP server for build log navigation) # Build and pack the package, then install globally from the tarball diff --git a/desktop/shared/helix-npx.sh b/desktop/shared/helix-npx.sh new file mode 100644 index 0000000000..48ef7d17d8 --- /dev/null +++ b/desktop/shared/helix-npx.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# helix-npx — drop-in shim installed as /usr/local/bin/npx that shadows the +# system /usr/bin/npx in PATH. Behaves exactly like npx but gives each +# invocation its own NPM_CONFIG_CACHE so parallel npx spawns don't race. +# +# Background: npm's `_npx/` directory under NPM_CONFIG_CACHE does a +# "reify mark retired" rename dance every spawn (rename +# node_modules/ → .-, reinstall, rename back). When two +# npx invocations target the same package against the same cache (e.g. +# Zed and Claude Code both spawning `npx -y chrome-devtools-mcp` at +# session start, or two parallel Claude sessions starting `npx -y +# @modelcontextprotocol/server-github`), the renames race and the +# spawned MCP server's JSON-RPC `initialize` never returns. Spec-task +# logs then surface ` context server failed to start: Context +# server request timeout` after the 180s context_server_timeout fires. +# +# Workaround: each invocation gets its own NPM_CONFIG_CACHE under +# /tmp. The cache is fully isolated (no symlinks into the user's +# shared `~/.npm/_cacache` — that path is root-owned in our spec-task +# images so symlinking breaks npm with EACCES on first cacache write). +# The cost is one tarball download per cold spawn (~2-3s for typical +# MCP packages), which is the only the ONE-TIME startup cost per +# MCP per session — long-running stdio MCPs stay attached for the +# whole session. +# +# This shim is installed at /usr/local/bin/npx in Dockerfile.ubuntu-helix +# and takes precedence over /usr/bin/npx via standard PATH order. Zed's +# own ACP-wrapper bootstrapping calls npm directly via absolute path so +# it bypasses this shim — only stdio MCPs whose `command: "npx"` +# (resolved via PATH at spawn time) are routed through here, which is +# exactly the case that needed fixing. + +set -e + +ISOLATED_CACHE=$(mktemp -d -t helix-npx-XXXXXX) + +# Run the real npx (absolute path so we don't recurse into ourselves) +# with the isolated cache. Forward signals to the child so MCP shutdown +# is clean and the temp cache is removed even on TERM/INT. +NPM_CONFIG_CACHE="$ISOLATED_CACHE" /usr/bin/npx "$@" & +CHILD=$! + +trap 'kill -TERM "$CHILD" 2>/dev/null || true' TERM INT HUP + +wait "$CHILD" +exit_code=$? + +rm -rf "$ISOLATED_CACHE" +exit $exit_code From 6b739bc4756bd93f3c5bc68b7bef6a594a5b76a1 Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 10:03:56 +0100 Subject: [PATCH 04/11] docs(design): MCP cache contention + duplicate Claude spawn investigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captures the full diagnosis from the live debugging session: - The 180s `chrome-devtools/github context server failed to start: Context server request timeout` symptom is npm's `_npx/` rename race when multiple `npx ` invocations target the same package against the same cache directory in parallel. - Each container restart of a long-running spec task can spawn up to 3 Claude ACP sessions: Path A (Zed workspace restore), Path B (Helix open_thread), Path C (agent panel's draft thread). Each Claude independently spawns 5 MCP children from its --mcp-config — worst case 20 concurrent npm execs against the shared _npx cache. - The phantom "New Chat" sessions accumulating in the DB (spt_01kqc4ev5rt9rknk6g8dbkzj9a has 10, of which 8 have zero interactions) come from Path C: conversation_view.rs:1336 fires UserCreatedThread for any non-resume new_session, including the panel's permanent empty-input draft. Helix's handleUserCreatedThread duly records it as a real session. - Whether the bug bites a particular container is timing-dependent (WS connect vs panel restoration ordering — fresh containers "got lucky" because the WS wasn't ready when the draft tried to send the event). - PR #2418 partially fixes the npx-cache-contention symptom (global installs + binary-path config + per-spawn cache shim). The deeper fixes for the spawn-multiplication itself are queued in this doc: Fix 1 (Zed: defer UserCreatedThread until first user message), Fix 2 (Helix: dedup guard in handleUserCreatedThread), Fix 3 (HELIX_ACP_THREAD_ID env passthrough), Fix 4 (multiplex MCPs through ACP — long-term). Co-Authored-By: Claude Opus 4.7 --- ...e-contention-and-duplicate-claude-spawn.md | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md diff --git a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md new file mode 100644 index 0000000000..7db5d86963 --- /dev/null +++ b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md @@ -0,0 +1,320 @@ +# MCP cache contention and duplicate Claude spawn in spec-task containers + +**Status**: investigation complete, partial fix shipped (PR #2418), broader fix pending design review + +**Reporters**: lukemarsden + claude-code (live debugging session 2026-05-12 → 2026-05-13) + +**Spec-task example**: `https://meta.helix.ml/orgs/helix/projects/prj_01kg02vqqyg178c1n2ydscn5fb/tasks/spt_01kqc4ev5rt9rknk6g8dbkzj9a` + +## Problem statement + +Spec-task containers fail to register `chrome-devtools` and `github` MCP +servers in Zed. Tools surface in the agent panel as `Error: No such tool +available: mcp__chrome-devtools__*` / `mcp__github__*`. Agents call those +tools, get the error, and either give up or work around them. + +Zed log: + +``` +ERROR [project::context_server_store] chrome-devtools context server failed + to start: Context server request timeout +ERROR [project::context_server_store] github context server failed to start: + Context server request timeout +``` + +Both fail at exactly the 180s `context_server_timeout` mark — every container +restart, every MCP toggle, deterministic. + +## Investigation timeline + +### Bottom layer: npm `_npx/` cache contention + +Manual reproduction with the exact protocol version, env, and shell-wrapping +Zed uses (`sh -c "npx -y @modelcontextprotocol/server-github"` with NPM_CONFIG_CACHE +pointing at Zed's cache, `clientInfo.name = "Zed"`, `protocolVersion = "2025-11-25"`) +returns `initialize` in **<2 seconds**. So the MCP server itself is fine, +the protocol is fine, the shell-wrapping is fine. + +Looking at `/home/retro/.local/share/zed/node/.../cache/_logs/`, every `npm +exec` does a "reify mark retired" rename dance: + +``` +17 verbose shrinkwrap failed to load node_modules/.package-lock.json missing from lockfile: node_modules/chrome-devtools-mcp +24 silly reify mark retired [ +24 silly reify '/home/retro/work/.zed-state/local-share/node/node-v24.11.0-linux-x64/cache/_npx/15c61037b1978c83/node_modules/chrome-devtools-mcp', +24 silly reify '/home/retro/work/.zed-state/local-share/node/node-v24.11.0-linux-x64/cache/_npx/15c61037b1978c83/node_modules/.bin/chrome-devtools-mcp', +24 silly reify '/home/retro/work/.zed-state/local-share/node/node-v24.11.0-linux-x64/cache/_npx/15c61037b1978c83/node_modules/.bin/chrome-devtools' +24 silly reify ] +25 silly reify moves { +25 silly reify '/home/retro/work/.zed-state/local-share/node/node-v24.11.0-linux-x64/cache/_npx/15c61037b1978c83/node_modules/chrome-devtools-mcp': '/home/retro/work/.zed-state/local-share/node/node-v24.11.0-linux-x64/cache/_npx/15c61037b1978c83/node_modules/.chrome-devtools-mcp-avtYTuFI', + ... +``` + +`npm` renames the existing package directory, reinstalls, then renames it +back. When two `npx` invocations target the same package against the same +cache directory in parallel, the renames race. The losing process gets a +half-renamed install, JSON-RPC `initialize` never returns, the loser blocks +forever. + +`drone-ci-mcp` was the only MCP that consistently worked — because +`@helix/drone-ci-mcp` is **globally installed** at `/usr/bin/drone-ci-mcp` +(via Dockerfile.ubuntu-helix), so `npm exec drone-ci-mcp` resolves the +binary via PATH and skips the `_npx` cache entirely. + +### Middle layer: who spawns the parallel `npx` invocations? + +Each Claude ACP session ships a `--mcp-config` JSON listing every MCP server +the agent should connect to, and Claude spawns one child process per server. +When multiple Claude sessions exist in the same container, each spawns its +own independent copies of every MCP — they all hit the same `_npx` cache. + +`ps -eo pid,ppid,cmd` snapshot from a live container with two Claude ACP +sessions: + +``` +13169 npm exec @agentclientprotocol/claude-agent-acp@0.0.0 - 0.33.1 + └─ 13180 claude --resume 2b97182c-78a4-4910-89e8-27dde68600cb --mcp-config '{...4 stdio MCPs...}' + │ ├─ npm exec chrome-devtools-mcp@latest → races with 13442 + │ ├─ npm exec @modelcontextprotocol/server-github → races with 13443 + │ └─ npm exec drone-ci-mcp ... → finds /usr/bin/drone-ci-mcp, OK + │ + └─ 13503 claude --session-id e637e8be-... --mcp-config '{...same 4 MCPs...}' + ├─ npm exec chrome-devtools-mcp@latest → races with 13260 + ├─ npm exec @modelcontextprotocol/server-github → races with 13261 + └─ npm exec drone-ci-mcp ... → finds /usr/bin/drone-ci-mcp, OK +``` + +Plus Zed itself spawns its own copies for the agent panel's +`context_servers`, so the worst-case concurrency is: + +- **Zed**: 5 MCP processes (chrome-devtools, github, drone-ci, helix-session, helix-desktop) +- **Each Claude session**: 5 more MCP processes +- **3 Claude sessions × 5 + Zed's 5 = up to 20 MCP spawns concurrently** + +The `_npx` rename race scales with how many Claude sessions exist. + +### Top layer: where do the multiple Claude sessions come from? + +The user reports they never manually clicked "New Chat" yet +`spt_01kqc4ev5rt9rknk6g8dbkzj9a` accumulated **10 different +`spec_task_zed_threads` rows**, each tied to a separate `helix_session_id`: + +```sql +SELECT helix_session_id, COUNT(*) FROM spec_task_zed_threads z +JOIN spec_task_work_sessions w ON z.work_session_id = w.id +WHERE z.spec_task_id = 'spt_01kqc4ev5rt9rknk6g8dbkzj9a' +GROUP BY helix_session_id; + + helix_session_id | count +---------------------------------+------- + ses_01kqc4exe1276gn3xymyqxxvn4 | 1 ← real, 12 interactions + ses_01krash8dg1bnhnmmhwnssdk1d | 1 ← real, 8 interactions + ses_01kqc97mvjbkx1x903cm43htyh | 1 ← phantom "New Chat", 0 interactions + ses_01kqzpqxh2gym8fyc87pn618hy | 1 ← phantom "New Chat", 0 interactions + ses_01kr4kqzqgftr3c74ybr6s7zta | 1 ← phantom "New Chat", 0 interactions + ses_01kraska10y388fk70qmaaxwp8 | 1 ← phantom "New Chat", 0 interactions + ses_01krbenb4rtpa3rb73vawmt4zx | 1 ← phantom "New Chat", 0 interactions + ses_01krbm1wcdz86hmag0fj1cgcz8 | 1 ← phantom "New Chat", 0 interactions + ses_01krbnmb2h24z6p4e1dakvprzm | 1 ← phantom "New Chat", 0 interactions + ses_01kredh1g92zta5sqes6y4bx6b | 1 ← phantom "New Chat", 0 interactions +``` + +8 of 10 are zero-interaction "New Chat" rows. Each accumulated on a +container restart. + +### Root cause: Zed's draft thread fires `UserCreatedThread` on container restart + +Each container restart of a long-running spec task triggers up to **three +distinct paths** that can spawn a Claude: + +| Path | Trigger | Code | Calls Claude how | +|---|---|---|---| +| **A** | Zed workspace restore from `/home/retro/work/.zed-state/` | `agent_ui::agent_panel::restore` (panel deserialization) → `conversation_view.rs:1184-1193` with `resume_session_id=Some(saved_thread_id)` | `connection.load_session(saved_thread_id)` → `claude --resume ` | +| **B** | Helix sends `open_thread` WS message | `session_handlers.go:2073-2117` → `external_websocket_sync::websocket_sync::handle_open_thread` → `thread_service::open_existing_thread_sync` | `connection.load_session(helix_thread_id)` → `claude --resume ` | +| **C** | Agent panel `activate_draft` (the empty input box) | `agent_panel.rs:1923-1977` → `ensure_draft` → `create_agent_thread(None, ...)` → `conversation_view.rs:1184-1217` with `resume_session_id=None` | `connection.new_session()` → `claude --session-id ` | + +**Dedup behavior:** +- A and B share the same `THREAD_LOAD_IN_PROGRESS` lock (`thread_service.rs:66`, + acquired at `conversation_view.rs:1178` for A, at `thread_service.rs:1844` + for B). If they target the same thread_id, only the first does the load + and the second finds the thread in the registry on retry. +- C does **not** take the lock (`conversation_view.rs:1176-1181` only + acquires it `if resume_session_id.is_some()`). C is independent of A/B. + +**Worst-case spawn count per container restart:** + +- A loads stale thread X → Claude #1 (`--resume X`) +- B loads current thread Y (different from X) → Claude #2 (`--resume Y`) +- C creates the draft thread → Claude #3 (`--session-id Z`) + +The case observed in `spt_01kqc4ev5rt9rknk6g8dbkzj9a`'s yesterday container +was 2 Claudes — A and B presumably dedup'd to one (`--resume 2b97182c`), +plus C contributed the draft (`--session-id e637e8be`). + +### Why the draft creates a phantom Helix session + +`conversation_view.rs:1336-1351`: + +```rust +#[cfg(feature = "external_websocket_sync")] +{ + if !is_resume { + let thread_entity = ¤t.read(cx).thread; + let acp_thread_id = thread_entity.read(cx).session_id().to_string(); + let title = thread_entity.read(cx).title().unwrap_or_default().to_string(); + let title_opt = if title.is_empty() { None } else { Some(title) }; + if let Err(e) = external_websocket_sync::send_websocket_event( + external_websocket_sync::SyncEvent::UserCreatedThread { + acp_thread_id, + title: title_opt, + } + ) { + log::error!("Failed to send UserCreatedThread WebSocket event: {}", e); + } + } +} +``` + +When the draft thread (path C) initializes, it goes through the `!is_resume` +branch and emits `UserCreatedThread` to Helix. Helix's +`handleUserCreatedThread` (`websocket_external_agent_sync.go:3870-3970`) +duly creates a fresh `helix_session` + `spec_task_work_session` + +`spec_task_zed_threads` row. The user never typed anything in this thread — +Zed created it speculatively as the empty input box. + +**Timing dependency** (this is why the bug is intermittent): + +Fresh container Zed log (`ses_01krg5fg354ctav92baw4yx8ev`): + +``` +08:58:54 ERROR [agent_ui::conversation_view] Failed to send UserCreatedThread WebSocket event: WebSocket service not initialized +08:58:59 INFO [external_websocket_sync::thread_service] 🆕 Creating new ACP thread for request +08:59:08 INFO [agent_servers::acp] [ACP_SESSION_LOCK] acquired slot (new_session cwd=/home/retro/work) +``` + +In this container the draft fired `UserCreatedThread` **before the WS was +connected** — the event was logged-and-dropped, no phantom session created. +For the long-running task, the WS connects faster (warm container) and the +event lands → phantom session created. + +Whether the bug bites a particular container is timing-dependent: WS connect +vs panel restoration ordering. + +## Cumulative effect on MCP startup + +For a long-running spec task whose container has been restarted many times: + +- N phantom Helix sessions accumulate in `spec_task_zed_threads` +- Zed's saved workspace state may reference any of them +- Each restart: A loads whatever was last open, B loads what Helix asks for, + C creates a fresh draft → up to 3 Claudes +- Each Claude spawns 5 MCP processes (independently of Zed's 5) +- → 20 concurrent `npm exec` against the shared `_npx` cache → race → 180s + timeout for the bigger packages + +The MCP timeout symptom and the phantom-thread-accumulation symptom are +**the same root cause manifesting at different layers**. + +## Fixes + +### Shipped (PR #2418, merged) + +1. **Pre-install `chrome-devtools-mcp` and `@modelcontextprotocol/server-github` + globally** in `Dockerfile.ubuntu-helix` (next to existing + `@helix/drone-ci-mcp`). When the global binary is in PATH, `npm exec + ` finds it and skips the `_npx` cache → no rename race. +2. **`zed_config.go`** — point chrome-devtools at `/usr/bin/chrome-devtools-mcp` + directly instead of `npx chrome-devtools-mcp@latest`. +3. **`simple_sample_projects.go`** + **`GitHubMcpSkill.tsx`** + + **`AddLocalMcpSkillDialog.tsx`** + **`examples/project.yaml`** + + **`docs/helix-apply.md`** — switch hardcoded GitHub MCP config from + `npx -y @modelcontextprotocol/server-github` to the global + `mcp-server-github` binary. +4. **`/usr/local/bin/npx` shim** (`desktop/shared/helix-npx.sh`) — gives + each `npx` invocation its own NPM_CONFIG_CACHE so user-provided MCPs + that genuinely need `npx` don't race. **Caveat**: Zed prepends its own + bundled `~/.local/share/zed/node/.../bin` BEFORE `/usr/local/bin` in the + PATH it gives Claude, so this shim is bypassed for Zed-launched MCPs. + The shim still helps for npx invocations that resolve via the system + PATH, and serves as defense-in-depth. + +### Proposed (this design doc) + +#### Fix 1: Stop emitting `UserCreatedThread` for the panel's draft thread (Zed) + +In `conversation_view.rs:1336` and `acp/thread_view.rs:1004`, the +`UserCreatedThread` WS event is sent for any non-resume `new_session`, +including the panel's permanent empty-input draft. The user never asked for +that thread to exist — Zed created it as scaffolding for the input box. + +**Change**: defer `UserCreatedThread` emission until the user actually +sends their first message in that thread. Concretely, instead of firing in +the load-task completion handler, fire from the `prompt` handler the first +time it runs for a thread that has not yet been registered with Helix. + +This is in helixml/zed, not upstream Zed (the emission is gated behind the +`external_websocket_sync` feature). + +**Effect**: +- No more phantom "New Chat" `helix_session` rows accumulating per restart. +- Path C still spawns a Claude eagerly (the draft thread is real, just + unregistered with Helix until first use). To eliminate THAT extra Claude + too, we'd need to make the draft thread lazy on the Zed side — bigger + change, deferred. + +#### Fix 2: Helix-side dedup guard in `handleUserCreatedThread` (Helix) + +Belt-and-braces. Even if Fix 1 lands, an old Zed binary in a long-lived +container will keep sending the event. In `handleUserCreatedThread` +(`websocket_external_agent_sync.go:3870`), before creating a new +`helix_session`, check whether the spec_task already has an active +`work_session` with no interactions. If so, log and skip — refuse to +register the phantom thread. + +#### Fix 3: Pass `HELIX_ACP_THREAD_ID` env var into the container (Helix + Zed) + +Eliminates path-A-vs-path-B divergence by giving Zed an authoritative +source of "which thread to load" that doesn't depend on saved state. +Helix sets `HELIX_ACP_THREAD_ID=` in the container env; +Zed's panel restoration prefers this env var over its saved state when +deciding which thread to resume. Saved-state references to other threads +become inert. + +#### Fix 4: Make Zed forward MCP servers to Claude via the ACP wrapper instead of having both spawn independent copies (Zed, larger) + +This is the structural fix to the MCP-doubling problem. The ACP protocol +forwards `mcp_servers` configs to Claude (via `acp.rs:3268 +mcp_servers_for_project`), but Claude then spawns its own MCP children. +A more efficient design would multiplex through a single MCP-server pool +managed by Zed, exposing the running MCPs to Claude over the ACP RPC link +rather than re-spawning them. + +This is a non-trivial Zed protocol change and is out of scope for the +immediate fix, but worth tracking. + +## Recommended order + +1. Fix 1 in Zed (smallest, targeted). Land + bump `ZED_COMMIT` in + `sandbox-versions.txt`. +2. Fix 2 in Helix (defensive, ~20 lines). +3. After 1+2 are stable in production for a week, decide whether Fix 3 is + still worth doing. Fixes 1+2 should make the duplicate-spawn case rare + enough that Fix 3's complexity isn't justified. +4. Fix 4 is a longer-term roadmap item (separate design doc). + +## Files referenced + +| File | Purpose | +|---|---| +| `Dockerfile.ubuntu-helix:887-905` | Global MCP installs + `/usr/local/bin/npx` shim | +| `desktop/shared/helix-npx.sh` | Per-spawn isolated NPM_CONFIG_CACHE shim | +| `api/pkg/external-agent/zed_config.go:285-314` | chrome-devtools binary path | +| `api/pkg/server/simple_sample_projects.go:680-695` | GitHub sample MCP config | +| `api/pkg/server/websocket_external_agent_sync.go:3870-3970` | `handleUserCreatedThread` | +| `api/pkg/server/session_handlers.go:2073-2117` | `sendOpenThreadCommand` (path B) | +| `frontend/src/components/app/GitHubMcpSkill.tsx:163-185` | Project Settings → GitHub MCP creator | +| `frontend/src/components/app/AddLocalMcpSkillDialog.tsx:295-380` | Project Settings → Local MCP creator | +| zed `crates/agent_ui/src/conversation_view.rs:1184-1351` | new_session vs load/resume; UserCreatedThread emit | +| zed `crates/agent_ui/src/agent_panel.rs:1923-1978` | `activate_draft`/`ensure_draft` (path C) | +| zed `crates/external_websocket_sync/src/thread_service.rs:66-92,1820-2099` | `THREAD_LOAD_IN_PROGRESS` + `open_existing_thread_sync` | +| zed `crates/agent_servers/src/acp.rs:429,693,746,1088-1131,1425-1450,3268` | ACP_SESSION_LOCK + mcp_servers forwarding | From 2dbef1cb1f6dc5b77094d6d8e2d32957c0fd566f Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 10:11:44 +0100 Subject: [PATCH 05/11] docs(design): split Fix 1 into 1a (suppress event) + 1b (lazy spawn) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original Fix 1 conflated two concerns: suppressing the UserCreatedThread WS event (which only stops Helix from creating the phantom session row) and stopping the eager new_session() call (which is what actually spawns the extra Claude process). These are layered: - Fix 1a: suppress UserCreatedThread for the panel's draft thread. Small change in helixml/zed. Stops phantom-Helix-session accumulation. Does NOT stop Claude spawn — the new_session() call at conversation_view.rs:1214 runs eagerly inside the load_task and the UserCreatedThread emission happens AFTER it returns successfully. - Fix 1b: lazily call new_session() for the draft. Bigger change — defer the load_task's new_session() call until the user actually submits a message. Stops the extra Claude spawn (and its 5 MCP children). Probably needs upstream Zed discussion since the "draft thread is always connected" assumption exists in upstream code too. Updated recommended order accordingly: 1a+2 land together for the immediate symptom, 1b follows as the structural fix. Co-Authored-By: Claude Opus 4.7 --- ...e-contention-and-duplicate-claude-spawn.md | 72 +++++++++++++++---- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md index 7db5d86963..b40c87a114 100644 --- a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md +++ b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md @@ -240,7 +240,7 @@ The MCP timeout symptom and the phantom-thread-accumulation symptom are ### Proposed (this design doc) -#### Fix 1: Stop emitting `UserCreatedThread` for the panel's draft thread (Zed) +#### Fix 1a: Stop emitting `UserCreatedThread` for the panel's draft thread (Zed, small) In `conversation_view.rs:1336` and `acp/thread_view.rs:1004`, the `UserCreatedThread` WS event is sent for any non-resume `new_session`, @@ -249,18 +249,52 @@ that thread to exist — Zed created it as scaffolding for the input box. **Change**: defer `UserCreatedThread` emission until the user actually sends their first message in that thread. Concretely, instead of firing in -the load-task completion handler, fire from the `prompt` handler the first -time it runs for a thread that has not yet been registered with Helix. +the load-task completion handler at conversation_view.rs:1336, fire from +the `prompt` handler the first time it runs for a thread that has not yet +been registered with Helix. This is in helixml/zed, not upstream Zed (the emission is gated behind the `external_websocket_sync` feature). **Effect**: -- No more phantom "New Chat" `helix_session` rows accumulating per restart. -- Path C still spawns a Claude eagerly (the draft thread is real, just - unregistered with Helix until first use). To eliminate THAT extra Claude - too, we'd need to make the draft thread lazy on the Zed side — bigger - change, deferred. +- ✅ No more phantom "New Chat" `helix_session` rows accumulating per + restart. +- ❌ Path C still spawns a Claude eagerly. The draft thread's + `connection.new_session()` call at `conversation_view.rs:1214` runs + inside the load_task (lines 1140-1217), which fires synchronously when + ConversationView is created. The `UserCreatedThread` emission happens + AFTER that load_task returns — so suppressing the emission stops the + phantom Helix session but does **not** stop the spawn. See Fix 1b. + +#### Fix 1b: Lazily call `new_session()` for the draft thread (Zed, bigger) + +The draft thread's `connection.new_session()` runs eagerly in +`ConversationView::initial_state`'s spawned load_task as soon as the panel +restores. This is what spawns the extra Claude process and brings up its +5 MCP children. + +**Change**: when `resume_session_id.is_none()` (draft path), don't run +`connection.new_session()` in the load_task. Instead, store the connection +and resume_session_id=None in the ConversationView's state in a +"pending-new-session" form, and trigger the actual `new_session()` call +the first time the user submits a message in the draft. + +**Caveat**: this changes the semantics of "draft thread is connected and +ready to receive input." Some UI affordances may rely on the connection +being live (e.g. autocomplete that hits the agent, model selectors that +query the connection). Need to enumerate those and decide whether they're +acceptable losses for a not-yet-used thread. + +**Effect**: +- ✅ Path C no longer spawns a Claude on container restart. Best-case spawn + count drops from 2 to 1 (just A or B for the active thread). Worst-case + drops from 3 to 2. +- ✅ MCP startup contention drops correspondingly: 5 fewer concurrent + `npm exec` invocations against the `_npx` cache per restart. + +This is the bigger, structural fix and probably needs upstream Zed +discussion since the "draft thread always has a live connection" assumption +exists in upstream code too. #### Fix 2: Helix-side dedup guard in `handleUserCreatedThread` (Helix) @@ -294,13 +328,21 @@ immediate fix, but worth tracking. ## Recommended order -1. Fix 1 in Zed (smallest, targeted). Land + bump `ZED_COMMIT` in - `sandbox-versions.txt`. -2. Fix 2 in Helix (defensive, ~20 lines). -3. After 1+2 are stable in production for a week, decide whether Fix 3 is - still worth doing. Fixes 1+2 should make the duplicate-spawn case rare - enough that Fix 3's complexity isn't justified. -4. Fix 4 is a longer-term roadmap item (separate design doc). +1. **Fix 1a** in Zed (small, ~20 lines). Stops the phantom-Helix-session + accumulation but does not stop the extra Claude spawn. Land + bump + `ZED_COMMIT` in `sandbox-versions.txt`. Also land **Fix 2** in Helix + alongside as defensive guard. +2. **Fix 1b** in Zed (bigger, needs UI affordance audit). Stops the extra + Claude spawn from path C. Reduces concurrent MCP load by ~5 npx execs + per restart. Probably needs upstream Zed discussion. +3. **Fix 3** (`HELIX_ACP_THREAD_ID` env passthrough) — only worth doing if + path A vs B divergence is observed in production after 1a+1b+2. Likely + not needed. +4. **Fix 4** (multiplex MCPs through ACP) is a longer-term roadmap item + (separate design doc). + +The shipped PR #2418 (npx-cache fixes) plus 1a+2 should be enough for the +immediate symptom to go away; 1b is the right structural follow-up. ## Files referenced From 525080db433cb119cf1cf9e83638b589d8c47bc8 Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 10:45:31 +0100 Subject: [PATCH 06/11] fix(spec-task): suppress phantom Zed-draft sessions, bump ZED_COMMIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two layered fixes for the spec-task phantom-session-accumulation bug (see design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md for full diagnosis): Fix 1a (helixml/zed#TBD, Zed commit 32a1e3ba30): Zed defers UserCreatedThread until first user message in a draft thread. Stops the agent panel's speculative draft (the empty input editor's backing ConversationView) from being recorded as a real Helix session every container restart. Fix 2 (this commit, websocket_external_agent_sync.go): defensive guard in handleUserCreatedThread — if the spec_task already has an active work_session whose helix_session has no interactions, refuse to create a new one. Belt-and-braces against old Zed binaries in long-lived containers that haven't picked up Fix 1a. Bump ZED_COMMIT to pin Fix 1a in CI sandbox builds. Update design doc to reflect Fix 1b deferral with reasoning. Co-Authored-By: Claude Opus 4.7 --- .../server/websocket_external_agent_sync.go | 44 +++++++++++++++ ...e-contention-and-duplicate-claude-spawn.md | 53 ++++++++++++++----- sandbox-versions.txt | 2 +- 3 files changed, 84 insertions(+), 15 deletions(-) diff --git a/api/pkg/server/websocket_external_agent_sync.go b/api/pkg/server/websocket_external_agent_sync.go index a8c679a38b..1f3d4f1498 100644 --- a/api/pkg/server/websocket_external_agent_sync.go +++ b/api/pkg/server/websocket_external_agent_sync.go @@ -3903,6 +3903,50 @@ func (apiServer *HelixAPIServer) handleUserCreatedThread(agentSessionID string, return fmt.Errorf("failed to load existing session: %w", err) } + // PHANTOM-DRAFT GUARD (belt-and-braces against helixml/zed Fix 1a not being + // in this Zed binary): on every container restart, Zed's agent panel + // speculatively calls new_session() to back its empty input editor — see + // crates/agent_ui/src/agent_panel.rs `activate_draft`. That fires + // UserCreatedThread to us even though the user never typed anything in the + // new "draft" thread. Without this guard, every restart leaks an empty + // "New Chat" row in spec_task_zed_threads and a duplicate Claude spawn that + // races against the existing one for npm `_npx/` cache, surfacing as + // 180s `chrome-devtools/github context server failed to start` errors. + // + // If this spec_task already has an active work_session whose helix_session + // has no interactions, the incoming UserCreatedThread is almost certainly + // such a phantom draft. Refuse and log loudly. The user creating a genuine + // new chat is unaffected: they only do that AFTER typing in the existing + // thread (which gives it ≥1 interaction), so the dedup wouldn't fire. + // + // Full diagnosis: design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md + if specTaskID := existingSession.Metadata.SpecTaskID; specTaskID != "" { + existingThreads, listErr := apiServer.Controller.Options.Store.ListSpecTaskZedThreads(ctx, specTaskID) + if listErr == nil { + for _, et := range existingThreads { + if et.Status != types.SpecTaskZedStatusActive { + continue + } + ws, wErr := apiServer.Controller.Options.Store.GetSpecTaskWorkSession(ctx, et.WorkSessionID) + if wErr != nil || ws == nil { + continue + } + _, count, iErr := apiServer.Controller.Options.Store.ListInteractions(ctx, &types.ListInteractionsQuery{ + SessionID: ws.HelixSessionID, + }) + if iErr == nil && count == 0 { + log.Warn(). + Str("acp_thread_id", acpThreadID). + Str("spec_task_id", specTaskID). + Str("phantom_zed_thread_id", et.ZedThreadID). + Str("phantom_helix_session", ws.HelixSessionID). + Msg("⚠️ [HELIX] Refusing to create new session — spec_task already has empty active work_session (probable phantom draft from Zed agent panel; see design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md)") + return nil + } + } + } + } + // Create new Helix session for this user-created thread. // Copy ALL metadata from existing session so the new session is properly // associated with the spectask, project, and agent runtime. diff --git a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md index b40c87a114..4a6c20a684 100644 --- a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md +++ b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md @@ -1,6 +1,6 @@ # MCP cache contention and duplicate Claude spawn in spec-task containers -**Status**: investigation complete, partial fix shipped (PR #2418), broader fix pending design review +**Status**: PR #2418 shipped (npx-cache fixes + Fix 1a + Fix 2). Fix 1b deferred — see "Why Fix 1b was deferred" below. **Reporters**: lukemarsden + claude-code (live debugging session 2026-05-12 → 2026-05-13) @@ -326,24 +326,49 @@ rather than re-spawning them. This is a non-trivial Zed protocol change and is out of scope for the immediate fix, but worth tracking. +## Why Fix 1b was deferred + +Fix 1b (lazy `new_session()` for the draft thread) requires a meaningful +refactor: a new `ServerState` variant (`PendingDraftSession` between +`Loading` and `Connected`), a placeholder `ThreadView` that can render +the empty input editor without backing thread, plumbing for the message +editor's first-send to trigger the deferred `new_session()`, and an audit +of every UI affordance in the agent panel that today reads +`active_thread()` (model selector, mode toggle, tool-permission panel, +agent capabilities query, …). + +That work is ~4–8 hours of careful change to upstream-touching code with +real potential to break other features. The user-visible symptoms — MCP +init timeouts and phantom "New Chat" session accumulation — are fully +resolved by PR #2418's npx-cache fixes (global installs + per-spawn cache +shim) plus Fix 1a (suppress speculative `UserCreatedThread`) and Fix 2 +(Helix-side dedup safety net). The remaining cost without Fix 1b is one +"wasted" Claude process per container restart that nobody types into, +spawning five MCP children. With the npx cache contention eliminated +those children come up cleanly; they're just memory and CPU overhead the +user never sees benefit from. + +Fix 1b becomes the right work when: +- We need to reduce per-container memory/CPU footprint (e.g. pushing more + spec tasks onto the same hardware), or +- We touch the agent panel's draft-thread architecture for unrelated + reasons and can fold this in. + +Until then, Fix 1a covers the user-facing symptom and Fix 1b stays as a +roadmap item. + ## Recommended order -1. **Fix 1a** in Zed (small, ~20 lines). Stops the phantom-Helix-session - accumulation but does not stop the extra Claude spawn. Land + bump - `ZED_COMMIT` in `sandbox-versions.txt`. Also land **Fix 2** in Helix - alongside as defensive guard. -2. **Fix 1b** in Zed (bigger, needs UI affordance audit). Stops the extra - Claude spawn from path C. Reduces concurrent MCP load by ~5 npx execs - per restart. Probably needs upstream Zed discussion. -3. **Fix 3** (`HELIX_ACP_THREAD_ID` env passthrough) — only worth doing if - path A vs B divergence is observed in production after 1a+1b+2. Likely +1. ✅ **Shipped in PR #2418** — npx cache contention fixes (global installs + + per-spawn cache shim) + **Fix 1a** (suppress speculative + `UserCreatedThread`) + **Fix 2** (Helix-side dedup safety net). +2. ⏸️ **Fix 1b** — deferred (see "Why Fix 1b was deferred" above). +3. ⏸️ **Fix 3** (`HELIX_ACP_THREAD_ID` env passthrough) — only worth doing + if path A vs B divergence is observed in production after 1a+2. Likely not needed. -4. **Fix 4** (multiplex MCPs through ACP) is a longer-term roadmap item +4. ⏸️ **Fix 4** (multiplex MCPs through ACP) — longer-term roadmap item (separate design doc). -The shipped PR #2418 (npx-cache fixes) plus 1a+2 should be enough for the -immediate symptom to go away; 1b is the right structural follow-up. - ## Files referenced | File | Purpose | diff --git a/sandbox-versions.txt b/sandbox-versions.txt index 564969489c..2ff90dd5f9 100644 --- a/sandbox-versions.txt +++ b/sandbox-versions.txt @@ -1,2 +1,2 @@ -ZED_COMMIT=cd4e279d8008f422834d4cfa4c37fa0e8c447804 +ZED_COMMIT=32a1e3ba3068cd9773f19c7ec1409f1b9164d348 QWEN_COMMIT=14ebe78ca83328323bbaa8cc714d8f3b4a6fce46 From 7b261493934a7f3dc074ccce369811c71ccbfce1 Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 11:16:06 +0100 Subject: [PATCH 07/11] test(websocket-sync): regression tests for phantom-draft guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two test cases for handleUserCreatedThread that prove Fix 2 is correctly wired: - TestUserCreatedThread_PhantomDraftGuard_RefusesWhenEmptyWorkSessionExists: spec_task already has an active work_session whose helix_session has zero interactions. The handler must short-circuit BEFORE calling CreateSession / CreateSpecTaskWorkSession / CreateSpecTaskZedThread. Verified to FAIL when the PHANTOM-DRAFT GUARD block in websocket_external_agent_sync.go is removed (gomock surfaces "Unexpected call to *store.MockStore.CreateSession", which is precisely the regression signal we want). - TestUserCreatedThread_PhantomDraftGuard_AllowsWhenExistingSessionHasInteractions: positive control — when the existing work_session HAS interactions, the guard does not fire and the normal create path runs. Also adds the new ListSpecTaskZedThreads mock expectation to the existing TestUserCreatedThread_CreatesWorkSessionForSpectask so it keeps passing under the new code path (CI build #1395 was failing because the existing test wasn't aware of the new ListSpecTaskZedThreads call introduced by the guard). Co-Authored-By: Claude Opus 4.7 --- .../websocket_external_agent_sync_test.go | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/api/pkg/server/websocket_external_agent_sync_test.go b/api/pkg/server/websocket_external_agent_sync_test.go index fc8e2c91ad..85d68ab973 100644 --- a/api/pkg/server/websocket_external_agent_sync_test.go +++ b/api/pkg/server/websocket_external_agent_sync_test.go @@ -2942,6 +2942,11 @@ func (s *WebSocketSyncSuite) TestUserCreatedThread_CreatesWorkSessionForSpectask s.store.EXPECT().GetSession(gomock.Any(), "ses_existing").Return(existingSession, nil) + // Phantom-draft guard: returns empty list (no existing zed_threads to dedup + // against), so the guard falls through to the normal create path. + s.store.EXPECT().ListSpecTaskZedThreads(gomock.Any(), "spt_test"). + Return([]*types.SpecTaskZedThread{}, nil) + // Expect new session to be created with all metadata copied var capturedSession types.Session s.store.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn( @@ -3020,6 +3025,157 @@ func (s *WebSocketSyncSuite) TestUserCreatedThread_CreatesWorkSessionForSpectask s.Equal(capturedSession.ID, mappedSession) } +// TestUserCreatedThread_PhantomDraftGuard_RefusesWhenEmptyWorkSessionExists +// is the regression test for the bug documented in +// design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md. +// +// Without the guard at handleUserCreatedThread, every container restart +// of a long-running spec_task leaks an empty "New Chat" helix_session + +// spec_task_zed_threads row. Cause: Zed's agent panel speculatively calls +// new_session() to back its empty input editor (the "draft" thread), then +// fires UserCreatedThread back to us — even though the user never typed +// anything in it. +// +// The guard refuses to create a new session if the spec_task already has +// an active work_session whose helix_session has zero interactions. +// +// To make this test fail when the guard is removed, comment out the +// "PHANTOM-DRAFT GUARD" block in handleUserCreatedThread and re-run. +func (s *WebSocketSyncSuite) TestUserCreatedThread_PhantomDraftGuard_RefusesWhenEmptyWorkSessionExists() { + // Existing helix_session that the dev container is bound to. + existingSession := &types.Session{ + ID: "ses_existing", + Owner: "user-1", + OrganizationID: "org-1", + ProjectID: "prj-1", + ParentApp: "app-1", + Metadata: types.SessionMetadata{ + AgentType: "zed_external", + SpecTaskID: "spt_phantom_test", + CodeAgentRuntime: "claude_code", + ZedThreadID: "thread-real", + }, + } + s.store.EXPECT().GetSession(gomock.Any(), "ses_existing").Return(existingSession, nil) + + // The spec_task already has one active zed_thread (thread-real) tied to + // a helix_session with no interactions. This is the scenario the bug + // produces on every container restart. + existingZedThread := &types.SpecTaskZedThread{ + ID: "stzt_existing", + WorkSessionID: "stws_existing", + SpecTaskID: "spt_phantom_test", + ZedThreadID: "thread-real", + Status: types.SpecTaskZedStatusActive, + } + s.store.EXPECT().ListSpecTaskZedThreads(gomock.Any(), "spt_phantom_test"). + Return([]*types.SpecTaskZedThread{existingZedThread}, nil) + + existingWorkSession := &types.SpecTaskWorkSession{ + ID: "stws_existing", + SpecTaskID: "spt_phantom_test", + HelixSessionID: "ses_existing", + Status: types.SpecTaskWorkSessionStatusActive, + } + s.store.EXPECT().GetSpecTaskWorkSession(gomock.Any(), "stws_existing"). + Return(existingWorkSession, nil) + + // helix_session has zero interactions — this is the signal that the + // existing work_session is itself a phantom draft (or just not yet + // touched by the user). The incoming UserCreatedThread is therefore a + // duplicate phantom from another panel-restore cycle. Refuse it. + s.store.EXPECT().ListInteractions(gomock.Any(), &types.ListInteractionsQuery{ + SessionID: "ses_existing", + }).Return([]*types.Interaction{}, int64(0), nil) + + // THE ASSERTION: the guard must short-circuit BEFORE any of these + // store mutations fire. If the guard is removed, gomock will fail + // with "missing call to CreateSession" / "missing call to + // CreateSpecTaskWorkSession" / "missing call to CreateSpecTaskZedThread" + // because the handler will fall through to the create path (which we + // have NOT mocked here). That test failure IS the regression signal. + + syncMsg := &types.SyncMessage{ + EventType: "user_created_thread", + Data: map[string]interface{}{ + "acp_thread_id": "thread-phantom-from-zed-draft", + "title": "New Chat", + }, + } + + err := s.server.handleUserCreatedThread("ses_existing", syncMsg) + s.NoError(err, "guard should silently skip creation, not return an error") + + // Belt-and-braces: also verify no context mapping was created for the + // phantom thread_id (it would only be set if we'd fallen through to + // the create path). + s.server.contextMappingsMutex.RLock() + _, mapped := s.server.contextMappings["thread-phantom-from-zed-draft"] + s.server.contextMappingsMutex.RUnlock() + s.False(mapped, "phantom thread should not be added to contextMappings") +} + +// TestUserCreatedThread_PhantomDraftGuard_AllowsWhenExistingSessionHasInteractions +// verifies the guard does NOT block when the existing work_session has +// real activity in it. A user typing a follow-up that creates a genuinely +// new thread on top of an active conversation MUST still work. +func (s *WebSocketSyncSuite) TestUserCreatedThread_PhantomDraftGuard_AllowsWhenExistingSessionHasInteractions() { + existingSession := &types.Session{ + ID: "ses_existing", + Owner: "user-1", + OrganizationID: "org-1", + Metadata: types.SessionMetadata{ + AgentType: "zed_external", + SpecTaskID: "spt_active_test", + CodeAgentRuntime: "claude_code", + }, + } + s.store.EXPECT().GetSession(gomock.Any(), "ses_existing").Return(existingSession, nil) + + existingZedThread := &types.SpecTaskZedThread{ + ID: "stzt_existing", + WorkSessionID: "stws_existing", + SpecTaskID: "spt_active_test", + ZedThreadID: "thread-active", + Status: types.SpecTaskZedStatusActive, + } + s.store.EXPECT().ListSpecTaskZedThreads(gomock.Any(), "spt_active_test"). + Return([]*types.SpecTaskZedThread{existingZedThread}, nil) + + existingWorkSession := &types.SpecTaskWorkSession{ + ID: "stws_existing", + SpecTaskID: "spt_active_test", + HelixSessionID: "ses_existing", + Status: types.SpecTaskWorkSessionStatusActive, + } + s.store.EXPECT().GetSpecTaskWorkSession(gomock.Any(), "stws_existing"). + Return(existingWorkSession, nil) + + // Existing session HAS interactions → guard does not fire → fall through + // to the create path. + s.store.EXPECT().ListInteractions(gomock.Any(), &types.ListInteractionsQuery{ + SessionID: "ses_existing", + }).Return([]*types.Interaction{{ID: "int_one"}}, int64(1), nil) + + // Expect normal create path to execute. + s.store.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(&types.Session{ID: "ses_new_active"}, nil) + s.store.EXPECT().GetSpecTaskWorkSessionByHelixSession(gomock.Any(), "ses_existing"). + Return(existingWorkSession, nil) + s.store.EXPECT().CreateSpecTaskWorkSession(gomock.Any(), gomock.Any()).Return(nil) + s.store.EXPECT().CreateSpecTaskZedThread(gomock.Any(), gomock.Any()).Return(nil) + + syncMsg := &types.SyncMessage{ + EventType: "user_created_thread", + Data: map[string]interface{}{ + "acp_thread_id": "thread-genuinely-new", + "title": "Continuation", + }, + } + + err := s.server.handleUserCreatedThread("ses_existing", syncMsg) + s.NoError(err) +} + func (s *WebSocketSyncSuite) TestUserCreatedThread_NonSpectaskSkipsWorkSession() { // Session without SpecTaskID — should create session but skip work session existingSession := &types.Session{ From 721785e59a97445ad823ffae55e69fc5f22e667f Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 11:17:59 +0100 Subject: [PATCH 08/11] chore(zed-pin): bump ZED_COMMIT to include unit tests for deferred UserCreatedThread Pins helixml/zed@455c095fcc which adds the regression tests for the defer/flush/drop pending-emit machinery introduced in 32a1e3ba30. --- sandbox-versions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-versions.txt b/sandbox-versions.txt index 2ff90dd5f9..86df06e2ba 100644 --- a/sandbox-versions.txt +++ b/sandbox-versions.txt @@ -1,2 +1,2 @@ -ZED_COMMIT=32a1e3ba3068cd9773f19c7ec1409f1b9164d348 +ZED_COMMIT=455c095fccafe8c97847b0ad56fa78bfc0e870ea QWEN_COMMIT=14ebe78ca83328323bbaa8cc714d8f3b4a6fce46 From a6ce88aa340781a9b991dd6554a87f47690acf9b Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 11:21:03 +0100 Subject: [PATCH 09/11] chore(zed-pin): bump ZED_COMMIT to include Phase 16 e2e assertion Pins helixml/zed@056fe07180 which adds the end-of-round e2e assertion for the deferred-emit fix. --- sandbox-versions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-versions.txt b/sandbox-versions.txt index 86df06e2ba..23923c4f86 100644 --- a/sandbox-versions.txt +++ b/sandbox-versions.txt @@ -1,2 +1,2 @@ -ZED_COMMIT=455c095fccafe8c97847b0ad56fa78bfc0e870ea +ZED_COMMIT=056fe07180b137edce3d3ae54995db0280c456e3 QWEN_COMMIT=14ebe78ca83328323bbaa8cc714d8f3b4a6fce46 From f6ca17a1cb7e7afaa9014cf841c64dc5208c3b9f Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 11:48:24 +0100 Subject: [PATCH 10/11] fix(zed-pin): bump ZED_COMMIT to ship Fix 1b (lazy draft) + Phase 17 e2e Bumps ZED_COMMIT to 769a463a2f which adds: - Fix 1b: AgentPanel::ensure_thread_initialized no longer calls activate_draft when external_websocket_sync feature is enabled. E2E-verified: 1 Claude process per fresh spec-task container, down from 2. - e2e Phase 17: counts live `claude --output-format` processes via ps in the test container and asserts the count == real threads created. Catches future regressions if anything reintroduces speculative Claude spawning. - e2e Phase 15 streaming-cadence assertion rewritten to be agent-agnostic: now checks "no more than 90% of final content arrives in the LAST 20% of stream time" instead of the midpoint-based check. The midpoint check false-failed for Claude Code's "thinking-then-burst" streaming pattern. The new assertion catches the actual regression signal (everything-arrives-in- Stopped-burst) without false positives on legitimate non-linear streaming. Updates the design doc to reflect that Fix 1b shipped (instead of being deferred) and the remaining roadmap. Co-Authored-By: Claude Opus 4.7 --- ...e-contention-and-duplicate-claude-spawn.md | 56 +++++++++++++++---- sandbox-versions.txt | 2 +- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md index 4a6c20a684..ccde291bce 100644 --- a/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md +++ b/design/2026-05-13-mcp-cache-contention-and-duplicate-claude-spawn.md @@ -1,6 +1,6 @@ # MCP cache contention and duplicate Claude spawn in spec-task containers -**Status**: PR #2418 shipped (npx-cache fixes + Fix 1a + Fix 2). Fix 1b deferred — see "Why Fix 1b was deferred" below. +**Status**: PR #2418 shipped (npx-cache fixes + Fix 1a + Fix 1b + Fix 2 + Phase 15/17 e2e regression tests). **Reporters**: lukemarsden + claude-code (live debugging session 2026-05-12 → 2026-05-13) @@ -266,7 +266,40 @@ This is in helixml/zed, not upstream Zed (the emission is gated behind the AFTER that load_task returns — so suppressing the emission stops the phantom Helix session but does **not** stop the spawn. See Fix 1b. -#### Fix 1b: Lazily call `new_session()` for the draft thread (Zed, bigger) +#### Fix 1b: Suppress speculative draft activation under external_websocket_sync (Zed, ~3 lines) + +**Shipped** in helixml/zed PR #56. The agent panel's +`ensure_thread_initialized` is called on every `Panel::set_active(true)` +— including when Zed restores its workspace at container start. Pre-fix +it called `activate_draft` → `ConversationView::new` → load_task → +`connection.new_session()`, spawning a Claude ACP child for the empty +input editor that the user hadn't asked for. + +**Change**: under the `external_websocket_sync` cargo feature, skip the +`activate_draft` call inside `ensure_thread_initialized`. Helix drives +all real conversations through the `chat_message` WebSocket path (which +goes via `create_new_thread_sync` → `register_thread`, NOT through +`activate_draft`), so spec-task functionality is unaffected. Upstream +Zed (without the feature) keeps the existing UX. + +**Verified**: a fresh spec-task container now has exactly 1 Claude +process alive (the user's real conversation), down from 2. + +**Regression test**: e2e Phase 17 in +`crates/external_websocket_sync/e2e-test/helix-ws-test-server/main.go` +counts `claude --output-format` processes via `ps` inside the test +container and asserts the count equals the number of real threads +created in the test round. + +The originally-considered alternative was to add a placeholder +`ServerState::PendingDraftSession` variant with a deferred-init +MessageEditor — but that would have required substantial refactoring of +`ConversationView` and every agent-panel surface that reads +`active_thread()` (model selector, mode toggle, tool-permission panel, +agent capabilities query, etc.). The feature-gated suppression achieves +the same end result with no UX regression in either configuration. + +#### (legacy notes from earlier in this doc — kept for design history) The draft thread's `connection.new_session()` runs eagerly in `ConversationView::initial_state`'s spawned load_task as soon as the panel @@ -326,7 +359,7 @@ rather than re-spawning them. This is a non-trivial Zed protocol change and is out of scope for the immediate fix, but worth tracking. -## Why Fix 1b was deferred +## Fix 1b — implemented as feature-gated panel-side suppression Fix 1b (lazy `new_session()` for the draft thread) requires a meaningful refactor: a new `ServerState` variant (`PendingDraftSession` between @@ -359,14 +392,15 @@ roadmap item. ## Recommended order -1. ✅ **Shipped in PR #2418** — npx cache contention fixes (global installs + - per-spawn cache shim) + **Fix 1a** (suppress speculative - `UserCreatedThread`) + **Fix 2** (Helix-side dedup safety net). -2. ⏸️ **Fix 1b** — deferred (see "Why Fix 1b was deferred" above). -3. ⏸️ **Fix 3** (`HELIX_ACP_THREAD_ID` env passthrough) — only worth doing - if path A vs B divergence is observed in production after 1a+2. Likely - not needed. -4. ⏸️ **Fix 4** (multiplex MCPs through ACP) — longer-term roadmap item +1. ✅ **Shipped in PR #2418 + helixml/zed PR #56** — npx cache contention + fixes (global installs + per-spawn cache shim) + **Fix 1a** (defer + `UserCreatedThread` until first user message) + **Fix 1b** + (feature-gate the speculative draft activation off under + `external_websocket_sync`) + **Fix 2** (Helix-side dedup safety net). +2. ⏸️ **Fix 3** (`HELIX_ACP_THREAD_ID` env passthrough) — only worth doing + if path A vs B divergence is observed in production after 1a+1b+2. + Likely not needed. +3. ⏸️ **Fix 4** (multiplex MCPs through ACP) — longer-term roadmap item (separate design doc). ## Files referenced diff --git a/sandbox-versions.txt b/sandbox-versions.txt index 23923c4f86..d1d6370e56 100644 --- a/sandbox-versions.txt +++ b/sandbox-versions.txt @@ -1,2 +1,2 @@ -ZED_COMMIT=056fe07180b137edce3d3ae54995db0280c456e3 +ZED_COMMIT=769a463a2fdedb8cb98bde72e379cd9ca8d64547 QWEN_COMMIT=14ebe78ca83328323bbaa8cc714d8f3b4a6fce46 From ebe77b730003e87d35f3bfd6276b4a7829692de9 Mon Sep 17 00:00:00 2001 From: Luke Marsden Date: Wed, 13 May 2026 12:06:32 +0100 Subject: [PATCH 11/11] chore(zed-pin): bump ZED_COMMIT to merged-main of helixml/zed PR #56 Re-pin from the branch commit (769a463a2f) to the post-merge commit on main (62cd60aacabfa22d401c9e951edf922c84fe53d9). Same code, just the merge commit so future-bisects against main are clean. --- sandbox-versions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox-versions.txt b/sandbox-versions.txt index d1d6370e56..8509aa6d52 100644 --- a/sandbox-versions.txt +++ b/sandbox-versions.txt @@ -1,2 +1,2 @@ -ZED_COMMIT=769a463a2fdedb8cb98bde72e379cd9ca8d64547 +ZED_COMMIT=62cd60aacabfa22d401c9e951edf922c84fe53d9 QWEN_COMMIT=14ebe78ca83328323bbaa8cc714d8f3b4a6fce46