test(acp): record Haiku 5.5 qualification as FAIL

.aidw/claude-haiku-5-5-acp-qualification.md

@@ -0,0 +1,80 @@
+# Claude Haiku 5.5 ACP qualification
+
+- Repository: `geeen-jp/software-agent-sdk`
+- Issue: `#69`
+
+**Live verdict: FAIL — not qualified.**
+
+`failed_stage: same_run_acceptance_evidence`
+
+This is an evidence failure, not an observed provider incompatibility. The
+Runtime record lists five bounded probe runs, but their per-probe JSON does
+not establish every acceptance field for one session and completed turn. In
+particular, the same-turn output needed to prove structured-output acceptance
+and schema validity is inaccessible. The live tuple therefore cannot pass;
+no provider mismatch is inferred from the missing evidence.
+
+## Observed environment and run inventory
+
+- Claude Code host binary checked: **2.1.284**.
+- Repository `claude-agent-acp` version pin: **0.81.0**. This is the
+  repository pin, not per-probe JSON evidence of the resolved runtime version.
+- Runtime record: **five** bounded probe runs, at **01:42, 01:46, 01:50,
+  01:53, and 01:57 JST**.
+- SDK SHA recorded for the probe candidate:
+  `2b7c522c0d399e13899ef011108e6af55477b8c3`.
+
+These observations establish the host binary check, repository pin, run count,
+and candidate source identity. They do not fill missing acceptance fields in
+the individual probe JSON records.
+
+No credential values are included in this report.
+
+## Target tuple and live acceptance fields
+
+Target tuple: requested model `claude-haiku-5-5`, `effort=high`, read-only,
+Claude subscription/OAuth, and native `json_schema` output.
+
+| Field | Verdict | Missing same-run evidence |
+| --- | --- | --- |
+| Exact served model | FAIL — not proven | Provider-owned assistant model identity `claude-haiku-5-5` from the probed turn. The `haiku` selector alone is insufficient. |
+| High effort | FAIL — not proven | Accepted and effective `effort=high` configuration for the same session and turn. |
+| Read-only | FAIL — not proven | Advertised and confirmed read-only mode, re-proved after config options, for the probed session. |
+| Claude OAuth, no PAYG | FAIL — not proven | Subscription/OAuth authentication and absence of API-key, custom-base-URL, or PAYG configuration in the effective Claude environment. |
+| Native structured output | FAIL — not proven | Acceptance of the submitted `json_schema` format and a completed response that validates against that exact schema. |
+
+The qualification therefore fails at `same_run_acceptance_evidence`. A PASS
+requires one record linking the exact served model, effective effort,
+read-only proof, authentication path, and schema-valid response to the same
+session and turn. The five bounded runs do not change that result while the
+per-probe JSON lacks this linked evidence.
+
+## Local regression coverage
+
+The SDK tests cover Haiku 5.5 exact-model matching and mismatch, rejection of
+alias-only or missing same-turn model evidence, high-effort application and
+mismatch, read-only policy, resumed-session proof handling, OAuth isolation
+without PAYG API-key preference, and native schema metadata for fresh and
+resumed sessions. These tests do not establish that the live provider accepted
+the schema or returned a response that validates against it.
+
+## Source changes
+
+No production source changed. The candidate adds regression coverage for the
+existing generic Claude ACP path; the available evidence does not justify a
+Haiku-specific SDK branch or another provider-mechanics change.
+
+## Local verification
+
+- Focused ACP tests: **526 passed**, 6 warnings.
+- `make validate` on the code/test candidate: **PASS**.
+  - Sync: passed.
+  - SDK: 6,162 passed, 7 skipped, 10 xfailed, 671 warnings.
+  - Agent Server: 2,083 passed, 1,260 warnings.
+  - Workspace: 191 passed, 2 warnings.
+  - Cross: 438 passed, 1 skipped, 1 deselected, 317 warnings.
+  - Tools: 926 passed, 39 skipped, 1 warning.
+  - Pyright: 0 errors, 4 warnings.
+  - Pre-commit, `git diff --check`, and `git diff --exit-code`: passed.
+
+The canonical run validated the code/test tree before this report-only update.

tests/sdk/agent/test_acp_agent.py

@@ -112,6 +112,26 @@ def _structured_output_config() -> StructuredOutputConfig:
     )
 
 
+def _scope_judge_output_config() -> StructuredOutputConfig:
+    return StructuredOutputConfig(
+        mode="json_schema",
+        schema={
+            "type": "object",
+            "properties": {
+                "decision": {"type": "string", "enum": ["accept", "reject"]},
+                "rationale": {"type": "string", "minLength": 1},
+                "findings": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "maxItems": 5,
+                },
+            },
+            "required": ["decision", "rationale", "findings"],
+            "additionalProperties": False,
+        },
+    )
+
+
 def _message_chunk(text: str, message_id: str | None) -> AgentMessageChunk:
     return AgentMessageChunk(
         content=TextContentBlock(text=text, type="text"),
@@ -146,6 +166,12 @@ async def _verified_set_session_model(
 _CLAUDE_EXACT_MODEL_CASES = (
     pytest.param("claude-opus-5-5", "opus", "claude-opus-5-4", id="opus-5-5"),
     pytest.param("claude-sonnet-5-5", "sonnet", "claude-sonnet-5", id="sonnet-5-5"),
+    pytest.param(
+        "claude-haiku-5-5",
+        "haiku",
+        "claude-haiku-4-5-20251001",
+        id="haiku-5-5",
+    ),
 )
 
 
@@ -5085,6 +5111,24 @@ def test_exact_match_passes_after_provider_selector(self, model, selector):
         agent._verify_served_model_for_turn()
         assert agent._served_model_id == model
 
+    @pytest.mark.parametrize(
+        ("model", "selector"),
+        [
+            ("claude-haiku-5-5", "haiku"),
+            ("claude-sonnet-5-5", "sonnet"),
+        ],
+    )
+    def test_exact_request_rejects_alias_without_same_turn_model_evidence(
+        self, model, selector
+    ):
+        agent = self._agent(model=model, selector=selector)
+
+        with pytest.raises(ACPSessionModelError, match="served='UNKNOWN'"):
+            agent._verify_served_model_for_turn()
+
+        assert agent._served_model_id is None
+        assert agent.current_model_id == selector
+
     def test_passing_turn_publishes_model_and_failed_next_turn_restores_selector(
         self, tmp_path
     ):
@@ -5547,14 +5591,15 @@ def _qualified_claude_options(
         ]
 
     @pytest.mark.parametrize(
-        ("model", "selector"),
+        ("model", "selector", "effort"),
         [
-            ("claude-opus-5-5", "default"),
-            ("claude-sonnet-5-5", "default"),
+            ("claude-opus-5-5", "default", "medium"),
+            ("claude-sonnet-5-5", "default", "medium"),
+            ("claude-haiku-5-5", "default", "high"),
         ],
     )
     def test_qualified_claude_exact_model_defers_to_same_turn_evidence(
-        self, tmp_path, model, selector
+        self, tmp_path, model, selector, effort
     ):
         agent = _make_agent(
             acp_command=[
@@ -5564,7 +5609,7 @@ def test_qualified_claude_exact_model_defers_to_same_turn_evidence(
             ],
             acp_server="claude-code",
             acp_model=model,
-            acp_config_options={"effort": "medium"},
+            acp_config_options={"effort": effort},
             acp_permission_policy="read_only",
         )
         state = _make_state(tmp_path)
@@ -5588,16 +5633,66 @@ def test_qualified_claude_exact_model_defers_to_same_turn_evidence(
         assert agent._post_turn_model_verification_required is True
         assert agent.current_model_id == selector
         assert state.agent_state["acp_current_model_id"] == selector
+        conn.set_config_option.assert_awaited_once_with(
+            config_id="effort",
+            session_id="sess-new",
+            value=effort,
+        )
+
+    def test_haiku_high_effort_fails_when_config_response_reports_mismatch(
+        self, tmp_path
+    ):
+        agent = _make_agent(
+            acp_command=[
+                "npx",
+                "-y",
+                f"@agentclientprotocol/claude-agent-acp@{CLAUDE_AGENT_ACP_VERSION}",
+            ],
+            acp_server="claude-code",
+            acp_model="claude-haiku-5-5",
+            acp_config_options={"effort": "high"},
+            acp_permission_policy="read_only",
+        )
+        state = _make_state(tmp_path)
+        conn = _make_config_conn(
+            agent_name="claude-agent-acp",
+            agent_version=CLAUDE_AGENT_ACP_VERSION,
+            options=self._qualified_claude_options(),
+            modes=self._qualified_claude_modes(),
+        )
+        conn.set_config_option = AsyncMock(
+            return_value=SetSessionConfigOptionResponse(
+                config_options=[
+                    _config_option("model", "default", ["default", "opus", "sonnet"]),
+                    _config_option(
+                        "mode", "default", ["default", "plan", "bypassPermissions"]
+                    ),
+                    _config_option("effort", "medium", ["low", "medium", "high"]),
+                ]
+            )
+        )
+
+        with self._mocked_acp_runtime(agent, conn):
+            with pytest.raises(ACPSessionConfigError, match="after requesting 'high'"):
+                agent.init_state(state, on_event=lambda _: None)
+
+        conn.set_config_option.assert_awaited_once_with(
+            config_id="effort",
+            session_id="sess-new",
+            value="high",
+        )
+        conn.prompt.assert_not_called()
 
     @pytest.mark.parametrize(
-        ("model", "selector"),
+        ("model", "selector", "effort"),
         [
-            ("claude-opus-5-5", "default"),
-            ("claude-sonnet-5-5", "default"),
+            ("claude-opus-5-5", "default", "medium"),
+            ("claude-sonnet-5-5", "default", "medium"),
+            ("claude-haiku-5-5", "default", "high"),
         ],
     )
     def test_resume_preserves_historical_verified_model_until_fresh_turn(
-        self, tmp_path, model, selector
+        self, tmp_path, model, selector, effort
     ):
         agent = _make_agent(
             acp_command=[
@@ -5607,7 +5702,7 @@ def test_resume_preserves_historical_verified_model_until_fresh_turn(
             ],
             acp_server="claude-code",
             acp_model=model,
-            acp_config_options={"effort": "medium"},
+            acp_config_options={"effort": effort},
             acp_permission_policy="read_only",
         )
         state = _make_state(tmp_path)
@@ -5629,34 +5724,40 @@ def test_resume_preserves_historical_verified_model_until_fresh_turn(
         assert agent.current_model_id == model
         assert agent._selector_model_id == selector
         assert state.agent_state["acp_current_model_id"] == model
+        conn.set_config_option.assert_awaited_once_with(
+            config_id="effort",
+            session_id="sess-old",
+            value=effort,
+        )
 
         agent._reset_client_for_turn(None, lambda _: None, [], state=state)
         assert agent.current_model_id == selector
         assert state.agent_state["acp_current_model_id"] == selector
 
-    def test_claude_sonnet_alias_does_not_publish_exact_identity(self, tmp_path):
+    @pytest.mark.parametrize("alias", ["sonnet", "haiku"])
+    def test_claude_alias_does_not_publish_exact_identity(self, tmp_path, alias):
         agent = _make_agent(
             acp_command=[
                 "npx",
                 "-y",
                 f"@agentclientprotocol/claude-agent-acp@{CLAUDE_AGENT_ACP_VERSION}",
             ],
             acp_server="claude-code",
-            acp_model="sonnet",
+            acp_model=alias,
             acp_permission_policy="read_only",
         )
         state = _make_state(tmp_path)
         conn = _make_config_conn(
             agent_name="claude-agent-acp",
             agent_version=CLAUDE_AGENT_ACP_VERSION,
-            options=self._qualified_claude_options("sonnet"),
+            options=self._qualified_claude_options(alias),
             modes=self._qualified_claude_modes(),
         )
 
         with self._mocked_acp_runtime(agent, conn):
             agent.init_state(state, on_event=lambda _: None)
 
-        assert agent.current_model_id == "sonnet"
+        assert agent.current_model_id == alias
         assert agent._post_turn_model_verification_required is False
         assert agent._served_model_id is None
 
@@ -5688,12 +5789,13 @@ def test_read_only_selector_session_enables_raw_for_later_exact_switch(
         assert kwargs["claudeCode"]["emitRawSDKMessages"] is True
         assert agent._post_turn_model_verification_required is False
 
+    @pytest.mark.parametrize("model", ["claude-opus-5-5", "claude-haiku-5-5"])
     def test_qualified_claude_exact_model_requires_read_only_before_prompt(
-        self, tmp_path
+        self, tmp_path, model
     ):
         agent = _make_agent(
             acp_server="claude-code",
-            acp_model="claude-opus-5-5",
+            acp_model=model,
             acp_permission_policy="writable",
         )
         state = _make_state(tmp_path)
@@ -5741,7 +5843,10 @@ def test_unqualified_claude_exact_adapter_fails_closed_before_prompt(
         conn.prompt.assert_not_called()
         assert agent._initialized is False
 
-    @pytest.mark.parametrize("model", ["claude-opus-5-5", "claude-sonnet-5-5"])
+    @pytest.mark.parametrize(
+        "model",
+        ["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"],
+    )
     def test_qualified_claude_exact_model_resumes_without_reusing_selector_as_proof(
         self, tmp_path, model
     ):
@@ -7003,9 +7108,12 @@ def test_claude_fresh_session_sends_meta_and_protocol_model(self, tmp_path):
         )
         conn.set_config_option.assert_not_called()
 
-    @pytest.mark.parametrize("model", ["claude-opus-5-5", "claude-sonnet-5-5"])
+    @pytest.mark.parametrize(
+        "model",
+        ["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"],
+    )
     def test_claude_fresh_session_sends_structured_output_meta(self, tmp_path, model):
-        config = _structured_output_config()
+        config = _scope_judge_output_config()
         agent = _make_agent(
             acp_command=[
                 "npx",
@@ -7080,12 +7188,15 @@ def test_claude_resumed_session_preserves_structured_output_meta(self, tmp_path)
         )
         conn.new_session.assert_not_awaited()
 
-    @pytest.mark.parametrize("model", ["claude-opus-5-5", "claude-sonnet-5-5"])
+    @pytest.mark.parametrize(
+        "model",
+        ["claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"],
+    )
     def test_claude_resumed_session_combines_structured_output_and_model(
         self, tmp_path, model
     ):
         """Load carries output metadata and the requested model intent."""
-        config = _structured_output_config()
+        config = _scope_judge_output_config()
         agent = _make_agent(
             acp_command=[
                 "npx",

tests/sdk/agent/test_acp_claude_auth_isolation.py

@@ -875,7 +875,7 @@ def test_claude_oauth_is_not_exported_to_non_claude_providers(
             }
         ),
     )
-    captured = _capture_start_env(agent, tmp_path)
+    captured = _capture_start_env(agent, tmp_path, ignore_startup_error=True)
     assert CLAUDE_CREDENTIALS_SECRET_NAME not in captured
     assert CLAUDE_OAUTH_TOKEN_ENV not in captured
     assert captured.get("UNRELATED_TOKEN") == "keep-me"
@@ -1056,6 +1056,26 @@ def _claude_agent_with_token(model: str | None, **kwargs) -> ACPAgent:
     )
 
 
+def test_haiku_read_only_start_keeps_subscription_oauth_isolated(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    source = _write_host_oauth(tmp_path, monkeypatch)
+    before = source.read_text(encoding="utf-8")
+    agent = _claude_agent_with_token(
+        "claude-haiku-5-5",
+        acp_permission_policy="read_only",
+        acp_env={"ANTHROPIC_API_KEY": "sk-payg"},
+    )
+
+    captured = _capture_start_env(agent, tmp_path, ignore_startup_error=True)
+
+    assert captured[CLAUDE_OAUTH_TOKEN_ENV] == "fake-token"
+    assert "ANTHROPIC_API_KEY" not in captured
+    isolated = Path(captured["CLAUDE_CONFIG_DIR"])
+    assert not (isolated / CLAUDE_CREDENTIALS_FILENAME).exists()
+    assert source.read_text(encoding="utf-8") == before
+
+
 def test_sonnet_5_5_exact_model_gets_400k_auto_compact_window(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None: