fix(sandbox): keep Claude Code current so built-in model aliases resolve (#3136)

docs/guides/per-agent-models.md

@@ -109,12 +109,13 @@ in `orchestrator/agent_model_resolution.py` walks the chain:
 > **Minimum sandbox Claude Code version.** The `fable` / `fable[1m]`
 > aliases require Claude Code ≥ 2.1.170. The sandbox's
 > `CLAUDE_CODE_VERSION` build-arg in `sandbox/Dockerfile` defaults to
-> `stable`, which satisfies this on a fresh build. Deployments that
-> pin an older `CLAUDE_CODE_VERSION` will see refine/plan agent spawns
-> fail with an "unknown model" error from Claude Code — either bump
-> the pinned version or set a repo-level `default_agent_model: opus`
-> to opt back out of the fable default until the sandbox image is
-> rebuilt.
+> `latest` (#3137 — `stable` lagged the fable launch and crash-looped
+> refine/plan agents per #3136). A build-time gate in the Dockerfile
+> fails the image build if the installed binary doesn't know `fable`
+> or `opus`, so a stale pin surfaces at build time rather than at
+> spawn. If you need to pin to an older `CLAUDE_CODE_VERSION` that
+> predates the alias, also set a repo-level `default_agent_model: opus`
+> AND temporarily drop `fable` from the Dockerfile gate.
 
 The result is an `AgentModelDecision` dataclass with fields
 `(claude_code_alias: str, upstream: str, upstream_model: str | None,

docs/reference/agent-tools.md

@@ -545,7 +545,7 @@ complete shell CLI surface.
 
 ## Version pin
 
-`claude-agent-sdk` is pinned to `>=0.1.65,<0.2` in
+`claude-agent-sdk` is pinned to `>=0.2.97,<0.3` in
 `sandbox/pyproject.toml` and the `CLAUDE_AGENT_SDK_VERSION` ARG in
 `sandbox/Dockerfile`. A smoke test at
 `tests/sandbox/egg_agent_tools/test_sdk_surface.py` imports

sandbox/Dockerfile

@@ -230,28 +230,81 @@ RUN groupadd -g 1000 egg && \
     mkdir -p /home/egg/.config && chown egg:egg /home/egg/.config
 
 # Install claude-agent-sdk (changes more often than base deps, less often than CLI)
-# The egg script always passes the specific PyPI version to bust Docker's layer
-# cache when a new version is released. The "latest" fallback only applies if the
-# build-arg is omitted (e.g. manual docker build without the egg script).
-ARG CLAUDE_AGENT_SDK_VERSION=0.1.65
+# Nothing passes this build-arg anymore (the legacy egg CLI that resolved a
+# concrete PyPI version was removed in #1762), so the default below is the
+# operative pin — bumping it is also what busts Docker's layer cache so the
+# new release actually installs. Keep it in sync with the bounded range in
+# sandbox/pyproject.toml.
+ARG CLAUDE_AGENT_SDK_VERSION=0.2.97
 RUN if [ "$CLAUDE_AGENT_SDK_VERSION" = "latest" ]; then \
-        pip3 install --no-cache-dir --upgrade 'claude-agent-sdk>=0.1.65,<0.2'; \
+        pip3 install --no-cache-dir --upgrade 'claude-agent-sdk>=0.2.97,<0.3'; \
     else \
         pip3 install --no-cache-dir claude-agent-sdk==${CLAUDE_AGENT_SDK_VERSION}; \
     fi && \
     # Save installed version for update checks (non-fatal if this fails)
     (pip3 show claude-agent-sdk 2>/dev/null | grep '^Version:' | cut -d' ' -f2 > /opt/claude-agent-sdk-version.txt || true)
 
 # Install Claude Code CLI (native installer) for the egg user
-# The egg script always passes the specific npm version to bust Docker's layer
-# cache when a new version is released. The "stable" fallback only applies if the
-# build-arg is omitted (e.g. manual docker build without the egg script).
-ARG CLAUDE_CODE_VERSION=stable
+#
+# Channel choice: 'latest', not 'stable'. orchestrator/agent_model_resolution.py
+# defaults refine/plan roles to bare model-family aliases at launch ('fable'),
+# and the stable channel lags those launches — at the time of #3136 stable
+# (2.1.153) predated the fable alias entirely, so a stable-channel image
+# crash-loops every refine/plan agent at spawn.
+ARG CLAUDE_CODE_VERSION=latest
+# Cache-bust (#3136): BuildKit re-checks this URL's content on every build, so
+# the install layer below re-runs exactly when the release channel moves to a
+# new version. Without it the layer is keyed on the literal channel name
+# ('latest'/'stable') and the image's Claude Code freezes at whatever the
+# channel pointed to the first time the layer built.
+#
+# Caveats worth knowing if this ever silently regresses (#3137 review):
+#   - The URL is hardcoded to the 'latest' manifest regardless of the
+#     CLAUDE_CODE_VERSION arg. If the operator overrides the arg to
+#     'stable' or to a concrete version (e.g. '2.1.173'), the layer still
+#     rebuilds whenever 'latest' moves. For 'stable' this is wasteful but
+#     functionally fine — the install reruns and re-fetches the stable
+#     pointer. For a concrete version the reinstall is deterministic and
+#     idempotent. The wasteful-rebuild edge case is intentional: keeping
+#     the URL hardcoded keeps the cache-bust behaviour the same across
+#     channel-name args, which is the dominant path.
+#   - The cache-bust depends on the CDN preserving HTTP cache headers
+#     (verified: 'cache-control: public,no-cache,max-age=0' + 'ETag' +
+#     'Last-Modified', which BuildKit's 'ADD <URL>' uses as its cache
+#     key). If the CDN ever drops those headers (config change, hosting
+#     migration), the bust silently degrades to 'cached forever per URL
+#     string'. The alias gate further down catches that regression at
+#     build time as long as the 'fable' / 'opus' families haven't both
+#     also stagnated, but defense-in-depth: rebuild with --no-cache if
+#     you're debugging a stuck channel pointer.
+ADD https://downloads.claude.ai/claude-code-releases/latest /tmp/claude-code-latest-release
 RUN su - egg -c "curl -fsSL https://claude.ai/install.sh | bash -s -- $CLAUDE_CODE_VERSION" && \
+    rm -f /tmp/claude-code-latest-release && \
     # Fail the build if the CLI binary is missing (e.g. install script 403'd)
     test -x /home/egg/.local/bin/claude && \
     # Save installed version for update checks (non-fatal if this fails)
     (/home/egg/.local/bin/claude --version 2>/dev/null | head -1 > /home/egg/.local/VERSION || true) && \
+    # Fail the build if the installed build predates a model family that
+    # orchestrator/agent_model_resolution.py uses as a built-in default
+    # ('fable' for refine/plan roles, 'opus' for everything else): a Claude
+    # Code build whose alias table lacks the family rejects the bare alias at
+    # session init and the agent crash-loops at spawn (#3136). No auth is
+    # mounted at build time, so this is a strings-level heuristic — a build
+    # that knows a family alias embeds the versioned 'claude-<family>-*' IDs
+    # the alias maps to; a build that predates the family contains none.
+    for family in fable opus; do \
+        if ! grep -aq "claude-${family}-" /home/egg/.local/bin/claude; then \
+            echo "ERROR: installed Claude Code ($(cat /home/egg/.local/VERSION 2>/dev/null)) does not know the '${family}' model family."; \
+            echo "agent_model_resolution.py uses '${family}' as a built-in default — agents would crash-loop at spawn (#3136)."; \
+            echo "Pass a newer build: docker build --build-arg CLAUDE_CODE_VERSION=<version|latest> ..."; \
+            echo "If pinning to an older CC build is intentional (and every"; \
+            echo "agent_models entry has been pinned away from '${family}'),"; \
+            echo "drop '${family}' from the 'for family in ...' list above to"; \
+            echo "skip the gate for that family (matches the bypass guidance"; \
+            echo "in docs/guides/per-agent-models.md)."; \
+            exit 1; \
+        fi; \
+    done && \
     # Replace bundled ripgrep with system ripgrep to avoid ARM64 crashes on Asahi Linux
     CLAUDE_RG=$(find /home/egg/.local -name "rg" -type f 2>/dev/null | grep -E "ripgrep.*linux" | head -1) && \
     if [ -n "$CLAUDE_RG" ]; then \

sandbox/agent-config/rules/overseer.md

@@ -198,7 +198,7 @@ A producer may legitimately spend time reading, grepping, and exploring before e
 | `plan` | 3 minutes |
 | `implement` | 10 minutes |
 
-Active tool calls (file reads, grep, web searches, `Agent` spawns, `TodoWrite`) observed via `get_container_logs` during this window are **evidence of legitimate work**, not a stall. Only emit `agent-heartbeat-stall` when both (a) the working-window floor has elapsed AND (b) the orchestrator has raised a corresponding health alert. **Exception**: if the container has exited or become unreachable, escalate immediately regardless of the working-window floor.
+Active tool calls (file reads, grep, web searches, `Agent` spawns, `TaskCreate` / `TaskUpdate`, or the legacy `TodoWrite` on older Claude Code builds) observed via `get_container_logs` during this window are **evidence of legitimate work**, not a stall. Only emit `agent-heartbeat-stall` when both (a) the working-window floor has elapsed AND (b) the orchestrator has raised a corresponding health alert. **Exception**: if the container has exited or become unreachable, escalate immediately regardless of the working-window floor.
 
 ### Escalation triggers
 

sandbox/pyproject.toml

@@ -6,12 +6,14 @@ requires-python = ">=3.14"
 dependencies = [
     "pyyaml>=6.0",
     "requests>=2.31.0",
-    # Pin claude-agent-sdk to a bounded pre-1.0 range. create_sdk_mcp_server
-    # and @tool are confirmed on 0.1.65 — see sandbox/egg_agent_tools/. A
-    # 0.2 bump must be validated manually; see
+    # Pin claude-agent-sdk to a bounded pre-1.0 range. The full surface egg
+    # uses (create_sdk_mcp_server, @tool, query, ClaudeAgentOptions fields,
+    # CLIJSONDecodeError hierarchy + the #2804 buffer-overflow marker) is
+    # confirmed on 0.2.97 — see sandbox/egg_agent_tools/. A 0.3 bump must be
+    # validated manually; see
     # tests/sandbox/egg_agent_tools/test_sdk_surface.py for the import-time
-    # smoke test that will fail CI loudly on API drift. See #1765.
-    "claude-agent-sdk>=0.1.65,<0.2",
+    # smoke test that will fail CI loudly on API drift. See #1765, #3136.
+    "claude-agent-sdk>=0.2.97,<0.3",
     "cryptography>=41.0.0",
     "PyJWT>=2.8.0",
 ]

shared/egg_agent/client.py

@@ -195,6 +195,20 @@ async def run_agent_async(
     """
     model = model or DEFAULT_MODEL
 
+    # Pin MCP servers to blocking connect (#3137 review). The 0.2 SDK / CLI
+    # bump made the spawned Claude Code default ``MCP_CONNECTION_NONBLOCKING``
+    # to non-zero, so a slow stdio MCP server is reported as ``pending`` and
+    # its tools are not available on the model's first turn. egg's in-process
+    # SDK MCP servers (registered below) don't actually connect over stdio so
+    # the change does not affect them, but the egg-ddg stdio fallback
+    # registered for the LiteLLM→non-Anthropic path does — and the
+    # ``SYSTEM_PROMPT_NUDGE`` that steers tool discovery is load-bearing on
+    # the first turn. Force blocking-connect so DDG tools are reliably ready
+    # before the first model call; ``setdefault`` preserves an operator-set
+    # value if one is already on the env. Cheap to keep on for the in-process
+    # path too (no-op there).
+    os.environ.setdefault("MCP_CONNECTION_NONBLOCKING", "0")
+
     # Resolve cwd: explicit arg > EGG_REPO_PATH > SDK default (os.getcwd()).
     # Sandbox agents start at HOME (/home/egg) while the repo lives at
     # /home/egg/repos/<repo> (EGG_REPO_PATH).  Defaulting to EGG_REPO_PATH

shared/egg_overseer/advisor.py

@@ -17,7 +17,7 @@
 
 Implementation choice: **Option B (two-call pattern)** per the SDK
 spike recorded at ``.egg-state/agent-outputs/1962-sdk-spike.md``. The
-vendored ``claude-agent-sdk==0.1.65`` does not expose the native
+vendored ``claude-agent-sdk`` (0.1.65 at the time) did not expose the native
 ``advisor_20260301`` tool, so we issue a separate ``run_agent_async``
 call against the configured Opus model with a single-turn prompt that
 follows the ``decision-20`` opt-3 contract (distilled summary).

tests/sandbox/egg_agent_tools/test_sdk_surface.py

@@ -1,11 +1,12 @@
 """Symbol-level import smoke test for claude-agent-sdk.
 
 If the SDK version pinned in ``sandbox/pyproject.toml`` drops or renames
-either ``create_sdk_mcp_server`` or ``tool``, CI will fail at
-test-collection time with a message pointing at the SDK release notes.
+any of the symbols egg imports, CI will fail at test-collection time
+with a message pointing at the SDK release notes.
 
-This is TASK-6-1 from the plan: pin the SDK to ``>=0.1.65,<0.2`` and
-guard the surface used by ``sandbox/egg_agent_tools``.
+This is TASK-6-1 from the plan: pin the SDK to ``>=0.2.97,<0.3`` and
+guard the surface used by ``sandbox/egg_agent_tools`` and the
+sandbox-side wrapper ``shared/egg_agent/client.py``.
 
 The test is skipped outside the sandbox (where the SDK is not installed)
 so it behaves gracefully in CI — the real enforcement is the
@@ -16,6 +17,30 @@
 
 import pytest
 
+# Symbols imported by ``shared/egg_agent/client.py`` — a 0.3 bump that
+# renames or drops any of these must update this list AND the importer
+# in lockstep, or agent spawn breaks with ``ImportError`` at runtime.
+# Sourced from ``shared/egg_agent/client.py:219-235`` and the
+# ``CLIJSONDecodeError`` reference in the same block (#2804 marker).
+_EGG_AGENT_CLIENT_SYMBOLS = (
+    "AssistantMessage",
+    "ClaudeAgentOptions",
+    "ClaudeSDKError",
+    "CLIJSONDecodeError",
+    "CLINotFoundError",
+    "HookMatcher",
+    "PermissionResultAllow",
+    "PermissionResultDeny",
+    "ProcessError",
+    "ResultMessage",
+    "SystemMessage",
+    "TextBlock",
+    "ToolResultBlock",
+    "ToolUseBlock",
+    "UserMessage",
+    "query",
+)
+
 
 def test_sdk_exposes_create_sdk_mcp_server() -> None:
     try:
@@ -41,6 +66,27 @@ def test_sdk_exposes_tool_decorator() -> None:
     )
 
 
+@pytest.mark.parametrize("symbol", _EGG_AGENT_CLIENT_SYMBOLS)
+def test_sdk_exposes_egg_agent_client_symbols(symbol: str) -> None:
+    """Guard every symbol ``shared/egg_agent/client.py`` imports from the SDK.
+
+    A 0.3 release that drops any of these would crash-loop agent spawn
+    with ``ImportError`` at the top of ``run_agent_async`` — surface it
+    at CI time instead. Update both this list and the importer when
+    bumping the pin.
+    """
+    try:
+        import claude_agent_sdk
+    except ImportError:
+        pytest.skip("claude_agent_sdk not installed in this environment")
+
+    assert hasattr(claude_agent_sdk, symbol), (
+        f"claude-agent-sdk no longer exposes {symbol!r} — "
+        "check release notes, update shared/egg_agent/client.py, "
+        "and update the pin in sandbox/pyproject.toml"
+    )
+
+
 def test_sandbox_pyproject_pins_sdk() -> None:
     """Defend the pin in sandbox/pyproject.toml — a missing bound lets
     the SDK auto-upgrade past tested surface.  If this test fails,
@@ -53,6 +99,6 @@ def test_sandbox_pyproject_pins_sdk() -> None:
     # Match either a quoted string entry or a table entry.
     assert re.search(r"claude-agent-sdk[^\"'\n]*>=[^,]*,[^\"'\n]*<[^\"'\n]+", text), (
         "sandbox/pyproject.toml must pin claude-agent-sdk with a bounded "
-        "range like '>=0.1.65,<0.2' so the SDK cannot auto-upgrade past "
+        "range like '>=0.2.97,<0.3' so the SDK cannot auto-upgrade past "
         "tested surface"
     )

tests/shared/egg_agent/test_client.py

@@ -469,6 +469,29 @@ def test_empty_egg_repo_path_treated_as_unset(self, mock_query):
             options = mock_query.call_args.kwargs["options"]
             assert options.cwd is None
 
+    @patch("claude_agent_sdk.query", side_effect=_mock_query_success)
+    def test_mcp_connection_nonblocking_default(self, mock_query):
+        """``run_agent_async`` must set ``MCP_CONNECTION_NONBLOCKING=0`` on os.environ
+        before the SDK runs, so stdio MCP servers (e.g. the egg-ddg fallback
+        on the LiteLLM→non-Anthropic path) finish their handshake before the
+        first model turn — see #3137 for the SDK 0.2.x behavior shift."""
+        with patch.dict(os.environ, {}, clear=False) as env:
+            env.pop("MCP_CONNECTION_NONBLOCKING", None)
+            _run_async(run_agent_async("test prompt"))
+
+            assert os.environ.get("MCP_CONNECTION_NONBLOCKING") == "0"
+
+    @patch("claude_agent_sdk.query", side_effect=_mock_query_success)
+    def test_mcp_connection_nonblocking_preserves_operator_override(self, mock_query):
+        """``setdefault`` semantics: if an operator already set the var (e.g.
+        to ``1`` for debugging a slow MCP server), ``run_agent_async`` must
+        not clobber it. Preserving operator intent is a hard requirement of
+        the #3137 fix."""
+        with patch.dict(os.environ, {"MCP_CONNECTION_NONBLOCKING": "1"}):
+            _run_async(run_agent_async("test prompt"))
+
+            assert os.environ.get("MCP_CONNECTION_NONBLOCKING") == "1"
+
     @patch("claude_agent_sdk.query", side_effect=_mock_query_error)
     def test_structured_logging_on_error(self, mock_query):
         """Test that system/result log is emitted on error paths."""