diff --git a/crates/ironclaw_reborn_composition/src/extension_lifecycle.rs b/crates/ironclaw_reborn_composition/src/extension_lifecycle.rs index c13025dfddc..20fd4504916 100644 --- a/crates/ironclaw_reborn_composition/src/extension_lifecycle.rs +++ b/crates/ironclaw_reborn_composition/src/extension_lifecycle.rs @@ -222,7 +222,7 @@ impl RebornLocalExtensionManagementPort { ); if extension_search_has_installed_external_channel_result(response.payload.as_ref()) { response.message = Some( - "Search found installed external channel results. Search cannot prove the calling user's channel account is personally connected, so do not treat those results as ready for delivery or message access. Call builtin.extension_activate now for the matching extension id; activation surfaces the channel-specific pairing/setup instructions and, for proof-code flows, the user must paste the code into the WebChat connection panel rather than normal chat." + "Search found installed external channel results. Search cannot prove the calling user's channel account is personally connected. For an explicit connect, pair, authenticate, or account-access request, call builtin.extension_activate for the matching extension id so channel-specific pairing/setup instructions can be surfaced. For routine, trigger, or notification delivery, prefer the configured outbound delivery target when one is available; do not activate the channel just to send to an already configured delivery target." .to_string(), ); } else if extension_search_has_ready_result(response.payload.as_ref()) { @@ -1782,7 +1782,7 @@ mod tests { } #[tokio::test] - async fn extension_search_routes_active_external_channel_to_activation_pairing_flow() { + async fn extension_search_distinguishes_external_channel_connect_from_delivery() { let (_dir, _storage_root, facade, _active_registry, _installation_store) = extension_lifecycle_fixture_with_catalog_and_service( AvailableExtensionCatalog::from_packages(vec![fixture_external_channel_package( @@ -1824,11 +1824,11 @@ mod tests { let message = search.message.as_deref().expect("search guidance"); assert!( message.contains("external channel") - && message.contains("do not treat") + && message.contains("explicit connect") && message.contains("builtin.extension_activate") - && message.contains("WebChat connection panel") - && message.contains("rather than normal chat"), - "active external channel search should route models into activation/pairing, got: {message}" + && message.contains("outbound delivery target") + && message.contains("do not activate"), + "active external channel search should distinguish connect requests from delivery, got: {message}" ); assert!( !message.contains("Treat those results as ready"), diff --git a/crates/ironclaw_reborn_composition/src/extension_lifecycle_capabilities.rs b/crates/ironclaw_reborn_composition/src/extension_lifecycle_capabilities.rs index 502786b9612..e3daae89e2f 100644 --- a/crates/ironclaw_reborn_composition/src/extension_lifecycle_capabilities.rs +++ b/crates/ironclaw_reborn_composition/src/extension_lifecycle_capabilities.rs @@ -68,7 +68,7 @@ fn manifests() -> Result, ExtensionError> { Ok(vec![ lifecycle_manifest( EXTENSION_SEARCH_CAPABILITY_ID, - "Search the local Reborn extension catalog by extension, product, provider, or service name. The catalog includes host-bundled extensions that are not installed yet and installed extensions that are inactive. For connect, enable, install, or integrate requests, use this for discovery only, then continue with builtin.extension_install for the matching extension instead of asking the user to configure credentials from search results. If search returns an installed external channel, still call builtin.extension_activate so channel-specific pairing/setup instructions can be surfaced before claiming the channel is ready.", + "Search the local Reborn extension catalog by extension, product, provider, or service name. The catalog includes host-bundled extensions that are not installed yet and installed extensions that are inactive. For connect, enable, install, pair, authenticate, or integrate requests, use this for discovery only, then continue with builtin.extension_install or builtin.extension_activate for the matching extension instead of asking the user to configure credentials from search results. For routine, trigger, or notification delivery, prefer configured outbound delivery targets before activating an external channel.", vec![EffectKind::ReadFilesystem], PermissionMode::Allow, )?, @@ -604,6 +604,7 @@ mod tests { && search.description.contains("service name") && search.description.contains("discovery only") && search.description.contains("external channel") + && search.description.contains("outbound delivery targets") && search .description .contains(EXTENSION_ACTIVATE_CAPABILITY_ID) diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index 2d4a2c1df5a..18cc9170547 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -136,7 +136,7 @@ Expected result: Slack is connected""", "qa_3b_endpoint_status_live_chat": """In WebUI, ask IronClaw "check if near.ai returns a 200 status." Expected result: IronClaw reports the endpoint's current HTTP status""", - "qa_3c_endpoint_status_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, ping [endpoint URL] checking if it returns a 200 status and send result in a DM in slack" + "qa_3c_endpoint_status_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, ping [endpoint URL] checking if it returns a 200 status and send result in a DM in slack." Expected result: Routine created""", "qa_4a_gmail_connect": """In WebUI, ask IronClaw "connect to Gmail." Go through the flow w/ Gmail. Expected result: Gmail is connected""", @@ -144,7 +144,7 @@ Expected result: GitHub is connected""", "qa_4c_github_release_live_chat": """In WebUI, ask IronClaw "summarize the latest release from https://github.com/nearai/ironclaw." Expected result: summary of the most recent release""", - "qa_4d_github_release_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, check https://github.com/nearai/ironclaw for latest releases and send me a Slack message summarizing any new ones." + "qa_4d_github_release_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, check https://github.com/nearai/ironclaw for latest releases and send me a Slack DM summarizing any new ones." Expected result: Routine created""", "qa_5a_slack_connect": """In WebUI, ask IronClaw "connect to Slack." Go through the auth flow. Expected result: Slack is connected""", @@ -162,8 +162,8 @@ Expected result: ABC sheet has new rows for each near.ai inbound email""", "qa_6d_gmail_to_sheet_routine": """In WebUI, ask IronClaw, "Every 30 minutes, check my inbox and add any new emails from a near.ai address to my Google Sheet called ABC." Expected result: Routine created""", - "qa_7a_slack_product_channel_connect": """In WebUI, ask IronClaw "connect to Slack, using channel #product." Go through the flow -Expected result: Slack channel is connected""", + "qa_7a_slack_product_channel_connect": """Verify Slack DM delivery target preflight configuration before Slack workflow cases run. +Expected result: Slack DM delivery target is configured""", "qa_7b_sheets_connect": """In WebUI, ask IronClaw "connect to Google Sheets." Go through the auth flow. Expected result: Google Sheets is connected""", "qa_7c_slack_bug_logger_routine": """In WebUI, ask IronClaw "whenever I send a slack message starting with 'bug:', add it as a row to my bug logging Google Sheet." @@ -174,7 +174,7 @@ Expected result: Slack is connected""", "qa_8b_hn_keyword_live_chat": """In WebUI, ask IronClaw "search Hacker News for any recent posts mentioning 'IronClaw' or 'NEAR AI'." Expected result: IronClaw reports any matching HN posts""", - "qa_8c_hn_keyword_slack_routine": """In WebUI, ask IronClaw, "Every hour, check Hacker News for new posts mentioning 'IronClaw' or 'NEAR AI' and send a summary to Slack." + "qa_8c_hn_keyword_slack_routine": """In WebUI, ask IronClaw, "Every hour, check Hacker News for new posts mentioning 'IronClaw' or 'NEAR AI' and send a summary to Slack DM." Expected result: Routine created""", } @@ -200,11 +200,6 @@ def _qa_sheet_prompt(case_name: str) -> str: EXTENSION_INSTALL_CAPABILITY_ID = "builtin.extension_install" EXTENSION_ACTIVATE_CAPABILITY_ID = "builtin.extension_activate" OUTBOUND_DELIVERY_TARGETS_LIST_CAPABILITY_ID = "builtin.outbound_delivery_targets_list" -QA_7A_CHAT_CONNECT_CAPABILITY_IDS = [ - EXTENSION_SEARCH_CAPABILITY_ID, - EXTENSION_INSTALL_CAPABILITY_ID, - EXTENSION_ACTIVATE_CAPABILITY_ID, -] QA_7C_BUG_LOGGING_SHEET_TITLE = "bug logging Google Sheet" @@ -1049,6 +1044,31 @@ def _record_assistant_reply_wait_result( observed["semantic_judge"] = reply.semantic_judge +def _routine_confirmation_follow_up_for_text(text: str) -> str | None: + normalized = text.lower() + asks_for_timezone = "timezone" in normalized or "time zone" in normalized + asks_for_confirmation = any( + phrase in normalized + for phrase in ( + "confirm", + "go ahead", + "shall i", + "should i", + "would you like", + ) + ) + routine_context = any( + phrase in normalized + for phrase in ("routine", "trigger", "automation", "schedule", "cron") + ) + if routine_context and (asks_for_timezone or asks_for_confirmation): + return ( + "Yes, go ahead and create it. Use Europe/London (London time) " + "for the schedule." + ) + return None + + async def _live_chat_case( ctx: LiveQaContext, *, @@ -1059,6 +1079,7 @@ async def _live_chat_case( timeout: float = 120.0, extra_details: dict[str, object] | None = None, forbidden_text: list[str] | None = None, + routine_confirmation_follow_up: bool = False, ) -> ProbeResult: from playwright.async_api import expect @@ -1091,16 +1112,35 @@ async def action(page: object) -> None: prompt[:80], timeout=15000, ) - _record_assistant_reply_wait_result( - observed, - await _wait_for_assistant_reply( - page, - marker=marker, - required_text=required_text, - timeout=timeout, - semantic_goal=prompt, - ), + reply = await _wait_for_assistant_reply( + page, + marker=marker, + required_text=required_text, + timeout=timeout, + semantic_goal=prompt, ) + _record_assistant_reply_wait_result(observed, reply) + if routine_confirmation_follow_up: + follow_up = _routine_confirmation_follow_up_for_text(reply.text_excerpt) + if follow_up: + observed["routine_confirmation_follow_up_sent"] = follow_up + observed["routine_confirmation_initial_text_excerpt"] = ( + reply.text_excerpt + ) + await composer.fill(follow_up) + await composer.press("Enter") + await expect(page.locator("[data-testid='msg-user']").last).to_contain_text( # type: ignore[attr-defined] + follow_up[:80], + timeout=15000, + ) + follow_up_reply = await _wait_for_assistant_reply( + page, + marker=marker, + required_text=required_text, + timeout=timeout, + semantic_goal=f"{prompt}\n{follow_up}", + ) + _record_assistant_reply_wait_result(observed, follow_up_reply) if forbidden_text: text = str(observed["text_excerpt"]).lower() matches = [ @@ -1396,6 +1436,17 @@ async def _live_github_latest_release(owner: str, repo: str) -> dict[str, str]: "Accept": "application/vnd.github+json", "User-Agent": "ironclaw-reborn-webui-v2-live-qa", } + token = _first_env_value( + [ + "AUTH_LIVE_GITHUB_TOKEN", + "IRONCLAW_REBORN_GITHUB_TOKEN", + "LIVE_CANARY_GITHUB_TOKEN", + "GITHUB_TOKEN", + "GH_TOKEN", + ] + ) + if token: + headers["Authorization"] = f"Bearer {token[1]}" async with httpx.AsyncClient(timeout=20.0, follow_redirects=True, headers=headers) as client: response = await client.get(url) response.raise_for_status() @@ -1929,6 +1980,11 @@ def _slack_preflight(ctx: LiveQaContext) -> dict[str, object]: return slack +def _slack_connect_instructions_look_valid(instructions: str) -> bool: + text = instructions.lower() + return "message the slack app" in text or ("slack" in text and "pairing code" in text) + + async def _slack_connect_case(ctx: LiveQaContext, *, case_name: str) -> ProbeResult: from playwright.async_api import expect @@ -1978,10 +2034,10 @@ async def action(page: object) -> None: if not title: raise AssertionError(f"Slack connect action title missing: {personal!r}") instructions = str(action_body.get("instructions") or "") - if "Message the Slack app" not in instructions: + if not _slack_connect_instructions_look_valid(instructions): raise AssertionError(f"unexpected Slack connect instructions: {instructions!r}") await expect(page.locator("body")).to_contain_text(title, timeout=15000) # type: ignore[attr-defined] - await expect(page.locator("body")).to_contain_text("Message the Slack app", timeout=15000) # type: ignore[attr-defined] + await expect(page.locator("body")).to_contain_text("pairing code", timeout=15000) # type: ignore[attr-defined] observed["slack_display_name"] = personal.get("display_name") observed["slack_connect_title"] = title observed["slack_connect_instructions"] = instructions @@ -2952,6 +3008,7 @@ async def _routine_creation_case( required_text=required_text, timeout=180.0, extra_details=details, + routine_confirmation_follow_up=True, ) after_count = _trigger_record_count(ctx.reborn_home, count_name) result.details["trigger_records_after"] = after_count @@ -3007,11 +3064,7 @@ async def _slack_delivery_routine_case( f"{routine_instruction} The routine's final answer and Slack message must " f"include the exact marker {delivery_marker}. Create the routine now; do not " "run it immediately. During routine creation, do not perform the routine's " - "live check, web/search/HTTP lookup, or Slack send. Before calling trigger_create, " - "call builtin__outbound_delivery_targets_list, then call " - "builtin__outbound_delivery_target_set with the Slack target id returned by the " - "list tool; do not only mention the target in text. Then create the routine " - "definition. " + "live check, web/search/HTTP lookup, or Slack send. " f"In your final answer include the exact marker {creation_marker} and include " "the text routine." ), @@ -3509,11 +3562,6 @@ async def case_qa_7c_slack_bug_logger_routine(ctx: LiveQaContext) -> ProbeResult required_text=["trigger|routine|automation|cron|schedule|fires|watches", "bug"], prompt=_qa_sheet_prompt("qa_7c_slack_bug_logger_routine"), extensions=[ - { - "package_id": "slack", - "display_name": "Slack", - "required_tools": [], - }, { "package_id": "google-drive", "display_name": "Google Drive", @@ -3543,16 +3591,11 @@ async def case_qa_7c_slack_bug_logger_routine(ctx: LiveQaContext) -> ProbeResult async def case_qa_7a_slack_product_channel_connect(ctx: LiveQaContext) -> ProbeResult: - from playwright.async_api import expect - started = time.monotonic() case_name = "qa_7a_slack_product_channel_connect" - prompt = ( - 'In WebUI, ask IronClaw "connect to Slack for my configured DM delivery ' - 'target." Go through the flow\n' - "Expected result: Slack DM delivery target is connected" - ) - observed: dict[str, object] = {"chat_connect_prompt": prompt} + observed: dict[str, object] = { + "preflight": "Slack DM delivery target is configured before user-story workflow cases" + } try: slack = _slack_preflight(ctx) delivery_channel_id = _slack_delivery_channel_id(ctx) @@ -3579,40 +3622,6 @@ async def case_qa_7a_slack_product_channel_connect(ctx: LiveQaContext) -> ProbeR "Slack live QA delivery target must be a DM to the user; " f"got channel_id={delivery_channel_id!r}" ) - - async def action(page: object) -> None: - capability_ids = QA_7A_CHAT_CONNECT_CAPABILITY_IDS - baseline_statuses = _capability_run_statuses( - ctx.reborn_home, - capability_ids, - ) - baseline_completed = _completed_capability_counts(baseline_statuses) - observed["baseline_capability_statuses"] = baseline_statuses - await page.goto( - f"{ctx.base_url}/v2/?token={AUTH_TOKEN}", - wait_until="domcontentloaded", - ) # type: ignore[attr-defined] - composer = page.locator("[data-testid='chat-composer']") # type: ignore[attr-defined] - await expect(composer).to_be_visible(timeout=15000) - await composer.fill(prompt) - await composer.press("Enter") - await expect(page.locator("[data-testid='msg-user']").last).to_contain_text( # type: ignore[attr-defined] - prompt[:80], - timeout=15000, - ) - _record_assistant_reply_wait_result( - observed, - await _wait_for_assistant_reply( - page, - marker=None, - required_text=["slack", "dm|delivery|target|connected"], - timeout=180.0, - ), - ) - statuses = _capability_run_statuses(ctx.reborn_home, capability_ids) - observed["capability_statuses"] = statuses - - await _with_page(ctx.output_dir, case_name, action) return _result(case_name, True, started, observed) except Exception as exc: return _result( diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index ef88ea51284..bd1f6cf4bff 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -180,7 +180,10 @@ async def fake_fetch_webui_json(_page: object, path: str) -> dict[str, object]: "strategy": "inbound_proof_code", "action": { "title": "Slack account connection", - "instructions": "Message the Slack app, then enter the code here.", + "instructions": ( + "Message the IronClaw Reborn app in Slack to get a " + "pairing code, then paste it here." + ), }, }, ] @@ -259,7 +262,7 @@ async def fake_fetch_webui_json(_page: object, path: str) -> dict[str, object]: observed_expectations = [text for _selector, text, _timeout in expected_texts] self.assertIn("Channels", observed_expectations) self.assertIn("Slack account connection", observed_expectations) - self.assertIn("Message the Slack app", observed_expectations) + self.assertIn("pairing code", observed_expectations) self.assertNotIn("Connect Slack", observed_expectations) self.assertFalse(any("/v2/chat" in url for url, _wait in fake_page.gotos)) self.assertEqual( @@ -271,6 +274,24 @@ async def fake_fetch_webui_json(_page: object, path: str) -> dict[str, object]: "Slack account connection", ) + def test_slack_connect_instruction_validation_accepts_pairing_copy(self): + self.assertTrue( + run_live_qa._slack_connect_instructions_look_valid( + "Message the Slack app, then enter the code here." + ) + ) + self.assertTrue( + run_live_qa._slack_connect_instructions_look_valid( + "Message the IronClaw Reborn app in Slack to get a pairing code, " + "then paste it here." + ) + ) + self.assertFalse( + run_live_qa._slack_connect_instructions_look_valid( + "Connect the channel from settings." + ) + ) + def test_product_connect_cases_start_from_chat_then_verify_registry(self): captured_chat: dict[str, dict[str, object]] = {} captured_registry: dict[str, dict[str, object]] = {} @@ -446,9 +467,13 @@ def fake_capability_run_statuses(_reborn_home, capability_ids): def test_routine_creation_case_fails_when_no_trigger_is_created(self): captured_prompts: list[str] = [] + captured_follow_up_flags: list[bool] = [] async def fake_live_chat_case(_ctx, **kwargs): captured_prompts.append(kwargs["prompt"]) + captured_follow_up_flags.append( + kwargs.get("routine_confirmation_follow_up", False) + ) extra_details = kwargs.get("extra_details") or {} return run_live_qa.ProbeResult( provider="test", @@ -482,9 +507,30 @@ async def fake_live_chat_case(_ctx, **kwargs): self.assertFalse(result.success) self.assertEqual(captured_prompts, ["original sheet prompt"]) + self.assertEqual(captured_follow_up_flags, [True]) self.assertEqual(result.details["trigger_records_after"], 0) self.assertIn("did not add a trigger_record", result.details["error"]) + def test_routine_confirmation_follow_up_answers_timezone_confirmation(self): + text = ( + "I'll set up a trigger every 5 minutes and send a Slack DM. " + "I need a timezone for scheduling. Shall I go ahead and create this?" + ) + + self.assertEqual( + run_live_qa._routine_confirmation_follow_up_for_text(text), + "Yes, go ahead and create it. Use Europe/London (London time) " + "for the schedule.", + ) + + def test_routine_confirmation_follow_up_ignores_slack_pairing_gate(self): + text = ( + "Connect Slack. Message the IronClaw Reborn app in Slack to get a " + "pairing code, then paste it here." + ) + + self.assertIsNone(run_live_qa._routine_confirmation_follow_up_for_text(text)) + def test_routine_creation_case_can_preinstall_extensions(self): captured: dict[str, object] = {} @@ -921,6 +967,49 @@ def test_slack_dm_route_discovery_rejects_missing_real_route_user(self): self.assertEqual(result["error"], "missing_slack_route_user_id") self.assertIn("REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_USER_ID", result["required_env"]) + def test_live_github_latest_release_uses_configured_token(self): + captured: dict[str, object] = {} + + class FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return {"tag_name": "ironclaw-v0.test", "name": "Test release"} + + class FakeAsyncClient: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def __aenter__(self): + return self + + async def __aexit__(self, _exc_type, _exc, _tb): + return None + + async def get(self, url): + captured["url"] = url + return FakeResponse() + + fake_httpx = types.SimpleNamespace(AsyncClient=FakeAsyncClient) + with ( + patch.dict(os.environ, {"AUTH_LIVE_GITHUB_TOKEN": "ghs_live"}, clear=True), + patch.dict(sys.modules, {"httpx": fake_httpx}), + ): + release = asyncio.run( + run_live_qa._live_github_latest_release("nearai", "ironclaw") + ) + + self.assertEqual(release["tag_name"], "ironclaw-v0.test") + self.assertEqual( + captured["url"], + "https://api.github.com/repos/nearai/ironclaw/releases/latest", + ) + self.assertEqual( + captured["headers"]["Authorization"], + "Bearer ghs_live", + ) + def test_qa_7a_requires_dm_delivery_target(self): with ( patch.object( @@ -962,74 +1051,7 @@ def test_slack_delivery_channel_ignores_failed_route_discovery_channel_id(self): self.assertIsNone(run_live_qa._slack_delivery_channel_id(ctx)) - def test_qa_7a_connect_capabilities_match_chat_connect_flow(self): - self.assertEqual( - run_live_qa.QA_7A_CHAT_CONNECT_CAPABILITY_IDS, - [ - run_live_qa.EXTENSION_SEARCH_CAPABILITY_ID, - run_live_qa.EXTENSION_INSTALL_CAPABILITY_ID, - run_live_qa.EXTENSION_ACTIVATE_CAPABILITY_ID, - ], - ) - self.assertNotIn( - run_live_qa.OUTBOUND_DELIVERY_TARGETS_LIST_CAPABILITY_ID, - run_live_qa.QA_7A_CHAT_CONNECT_CAPABILITY_IDS, - ) - - def test_qa_7a_accepts_existing_dm_delivery_target_without_new_connect_capabilities(self): - class FakeLocator: - @property - def last(self): - return self - - async def fill(self, _text): - return None - - async def press(self, _key): - return None - - class FakePage: - async def goto(self, _url, **_kwargs): - return None - - def locator(self, _selector): - return FakeLocator() - - class FakeExpectation: - async def to_be_visible(self, **_kwargs): - return None - - async def to_contain_text(self, _text, **_kwargs): - return None - - capability_ids = run_live_qa.QA_7A_CHAT_CONNECT_CAPABILITY_IDS - baseline = {capability_id: ["completed"] for capability_id in capability_ids} - stale = {capability_id: ["completed"] for capability_id in capability_ids} - fresh = { - capability_id: ["completed", "completed"] - for capability_id in capability_ids - } - status_sequence = [baseline, stale] - - async def fake_with_page(_output_dir, _case_name, action): - await action(FakePage()) - - async def fake_wait_for_assistant_reply(_page, **_kwargs): - return run_live_qa.AssistantReplyWaitResult( - text_excerpt="Slack is connected", - semantic_judge_used=False, - semantic_judge_reason="literal_required_text_matched", - ) - - async def fake_approve_visible_tool_gate(_page): - return None - - async def fake_sleep(_seconds): - return None - - def fake_capability_run_statuses(_reborn_home, _capability_ids): - return status_sequence.pop(0) if status_sequence else fresh - + def test_qa_7a_accepts_existing_dm_delivery_target_without_chat_connect(self): with ( patch.object( run_live_qa, @@ -1040,33 +1062,20 @@ def fake_capability_run_statuses(_reborn_home, _capability_ids): }, ), patch.object(run_live_qa, "_slack_delivery_channel_id", return_value="D12345"), - patch.object(run_live_qa, "_with_page", side_effect=fake_with_page), - patch.object( - run_live_qa, - "_wait_for_assistant_reply", - side_effect=fake_wait_for_assistant_reply, - ), - patch.object( - run_live_qa, - "_approve_visible_tool_gate", - side_effect=fake_approve_visible_tool_gate, - ), patch.object( run_live_qa, - "_capability_run_statuses", - side_effect=fake_capability_run_statuses, + "_with_page", + side_effect=AssertionError("QA 7A should not open WebUI chat"), ), - patch.object(run_live_qa.asyncio, "sleep", side_effect=fake_sleep), - patch("playwright.async_api.expect", return_value=FakeExpectation()), ): result = asyncio.run( run_live_qa.case_qa_7a_slack_product_channel_connect(self._dummy_ctx()) ) self.assertTrue(result.success) - self.assertEqual(result.details["baseline_capability_statuses"], baseline) - self.assertEqual(result.details["capability_statuses"], stale) - self.assertEqual(result.details["text_excerpt"], "Slack is connected") + self.assertEqual(result.details["slack_delivery_target_kind"], "dm") + self.assertEqual(result.details["delivery_target_present"], True) + self.assertIn("preflight", result.details) def test_completed_capability_counts_ignore_stale_completed_runs(self): counts = run_live_qa._completed_capability_counts( @@ -1183,7 +1192,7 @@ async def fake_routine_creation_case( package_ids = [ extension["package_id"] for extension in captured_routine["extensions"] ] - self.assertEqual(package_ids, ["slack", "google-drive", "google-sheets"]) + self.assertEqual(package_ids, ["google-drive", "google-sheets"]) self.assertEqual( captured_routine["extra_details"]["bug_log_sheet_fixture"]["spreadsheet_id"], "sheet-123",