Repository navigation
fix(ci): stabilize Slack live QA harness #5632
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -136,15 +136,15 @@ | |||||
| Expected result: Slack is connected""", | ||||||
| "qa_3b_endpoint_status_live_chat": """In WebUI, ask IronClaw "check if near.ai returns a 200 status." | ||||||
| Expected result: IronClaw reports the endpoint's current HTTP status""", | ||||||
| "qa_3c_endpoint_status_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, ping [endpoint URL] checking if it returns a 200 status and send result in a DM in slack." | ||||||
| Expected result: Routine created""", | ||||||
| "qa_4a_gmail_connect": """In WebUI, ask IronClaw "connect to Gmail." Go through the flow w/ Gmail. | ||||||
| Expected result: Gmail is connected""", | ||||||
| "qa_4b_github_connect": """In WebUI, ask IronClaw "connect to GitHub." Go through the auth flow. | ||||||
| Expected result: GitHub is connected""", | ||||||
| "qa_4c_github_release_live_chat": """In WebUI, ask IronClaw "summarize the latest release from https://github.com/nearai/ironclaw." | ||||||
| Expected result: summary of the most recent release""", | ||||||
| "qa_4d_github_release_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, check https://github.com/nearai/ironclaw for latest releases and send me a Slack DM summarizing any new ones." | ||||||
| Expected result: Routine created""", | ||||||
| "qa_5a_slack_connect": """In WebUI, ask IronClaw "connect to Slack." Go through the auth flow. | ||||||
| Expected result: Slack is connected""", | ||||||
|
|
@@ -162,8 +162,8 @@ | |||||
| Expected result: ABC sheet has new rows for each near.ai inbound email""", | ||||||
| "qa_6d_gmail_to_sheet_routine": """In WebUI, ask IronClaw, "Every 30 minutes, check my inbox and add any new emails from a near.ai address to my Google Sheet called ABC." | ||||||
| Expected result: Routine created""", | ||||||
| "qa_7a_slack_product_channel_connect": """Verify Slack DM delivery target preflight configuration before Slack workflow cases run. | ||||||
| Expected result: Slack DM delivery target is configured""", | ||||||
| "qa_7b_sheets_connect": """In WebUI, ask IronClaw "connect to Google Sheets." Go through the auth flow. | ||||||
| Expected result: Google Sheets is connected""", | ||||||
| "qa_7c_slack_bug_logger_routine": """In WebUI, ask IronClaw "whenever I send a slack message starting with 'bug:', add it as a row to my bug logging Google Sheet." | ||||||
|
|
@@ -174,7 +174,7 @@ | |||||
| Expected result: Slack is connected""", | ||||||
| "qa_8b_hn_keyword_live_chat": """In WebUI, ask IronClaw "search Hacker News for any recent posts mentioning 'IronClaw' or 'NEAR AI'." | ||||||
| Expected result: IronClaw reports any matching HN posts""", | ||||||
| "qa_8c_hn_keyword_slack_routine": """In WebUI, ask IronClaw, "Every hour, check Hacker News for new posts mentioning 'IronClaw' or 'NEAR AI' and send a summary to Slack DM." | ||||||
| Expected result: Routine created""", | ||||||
| } | ||||||
|
|
||||||
|
|
@@ -200,11 +200,6 @@ def _qa_sheet_prompt(case_name: str) -> str: | |||||
| EXTENSION_INSTALL_CAPABILITY_ID = "builtin.extension_install" | ||||||
| EXTENSION_ACTIVATE_CAPABILITY_ID = "builtin.extension_activate" | ||||||
| OUTBOUND_DELIVERY_TARGETS_LIST_CAPABILITY_ID = "builtin.outbound_delivery_targets_list" | ||||||
| QA_7C_BUG_LOGGING_SHEET_TITLE = "bug logging Google Sheet" | ||||||
|
|
||||||
|
|
||||||
|
|
@@ -1049,6 +1044,31 @@ def _record_assistant_reply_wait_result( | |||||
| observed["semantic_judge"] = reply.semantic_judge | ||||||
|
|
||||||
|
|
||||||
| def _routine_confirmation_follow_up_for_text(text: str) -> str | None: | ||||||
| normalized = text.lower() | ||||||
| asks_for_timezone = "timezone" in normalized or "time zone" in normalized | ||||||
| asks_for_confirmation = any( | ||||||
| phrase in normalized | ||||||
| for phrase in ( | ||||||
| "confirm", | ||||||
| "go ahead", | ||||||
| "shall i", | ||||||
| "should i", | ||||||
| "would you like", | ||||||
| ) | ||||||
| ) | ||||||
| routine_context = any( | ||||||
| phrase in normalized | ||||||
| for phrase in ("routine", "trigger", "automation", "schedule", "cron") | ||||||
| ) | ||||||
| if routine_context and (asks_for_timezone or asks_for_confirmation): | ||||||
| return ( | ||||||
| "Yes, go ahead and create it. Use Europe/London (London time) " | ||||||
| "for the schedule." | ||||||
| ) | ||||||
| return None | ||||||
|
|
||||||
|
|
||||||
| async def _live_chat_case( | ||||||
| ctx: LiveQaContext, | ||||||
| *, | ||||||
|
|
@@ -1059,6 +1079,7 @@ async def _live_chat_case( | |||||
| timeout: float = 120.0, | ||||||
| extra_details: dict[str, object] | None = None, | ||||||
| forbidden_text: list[str] | None = None, | ||||||
| routine_confirmation_follow_up: bool = False, | ||||||
| ) -> ProbeResult: | ||||||
| from playwright.async_api import expect | ||||||
|
|
||||||
|
|
@@ -1091,16 +1112,35 @@ async def action(page: object) -> None: | |||||
| prompt[:80], | ||||||
| timeout=15000, | ||||||
| ) | ||||||
| reply = await _wait_for_assistant_reply( | ||||||
| page, | ||||||
| marker=marker, | ||||||
| required_text=required_text, | ||||||
| timeout=timeout, | ||||||
| semantic_goal=prompt, | ||||||
| ) | ||||||
| _record_assistant_reply_wait_result(observed, reply) | ||||||
| if routine_confirmation_follow_up: | ||||||
| follow_up = _routine_confirmation_follow_up_for_text(reply.text_excerpt) | ||||||
| if follow_up: | ||||||
| observed["routine_confirmation_follow_up_sent"] = follow_up | ||||||
| observed["routine_confirmation_initial_text_excerpt"] = ( | ||||||
| reply.text_excerpt | ||||||
| ) | ||||||
| await composer.fill(follow_up) | ||||||
| await composer.press("Enter") | ||||||
| await expect(page.locator("[data-testid='msg-user']").last).to_contain_text( # type: ignore[attr-defined] | ||||||
| follow_up[:80], | ||||||
| timeout=15000, | ||||||
| ) | ||||||
| follow_up_reply = await _wait_for_assistant_reply( | ||||||
| page, | ||||||
| marker=marker, | ||||||
| required_text=required_text, | ||||||
| timeout=timeout, | ||||||
| semantic_goal=f"{prompt}\n{follow_up}", | ||||||
| ) | ||||||
| _record_assistant_reply_wait_result(observed, follow_up_reply) | ||||||
| if forbidden_text: | ||||||
| text = str(observed["text_excerpt"]).lower() | ||||||
| matches = [ | ||||||
|
|
@@ -1396,6 +1436,17 @@ async def _live_github_latest_release(owner: str, repo: str) -> dict[str, str]: | |||||
| "Accept": "application/vnd.github+json", | ||||||
| "User-Agent": "ironclaw-reborn-webui-v2-live-qa", | ||||||
| } | ||||||
| token = _first_env_value( | ||||||
| [ | ||||||
| "AUTH_LIVE_GITHUB_TOKEN", | ||||||
| "IRONCLAW_REBORN_GITHUB_TOKEN", | ||||||
| "LIVE_CANARY_GITHUB_TOKEN", | ||||||
| "GITHUB_TOKEN", | ||||||
| "GH_TOKEN", | ||||||
| ] | ||||||
| ) | ||||||
| if token: | ||||||
| headers["Authorization"] = f"Bearer {token[1]}" | ||||||
| async with httpx.AsyncClient(timeout=20.0, follow_redirects=True, headers=headers) as client: | ||||||
| response = await client.get(url) | ||||||
| response.raise_for_status() | ||||||
|
|
@@ -1929,6 +1980,11 @@ def _slack_preflight(ctx: LiveQaContext) -> dict[str, object]: | |||||
| return slack | ||||||
|
|
||||||
|
|
||||||
| def _slack_connect_instructions_look_valid(instructions: str) -> bool: | ||||||
| text = instructions.lower() | ||||||
| return "message the slack app" in text or ("slack" in text and "pairing code" in text) | ||||||
|
Comment on lines
+1983
to
+1985
There was a problem hiding this comment. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '\n== run_live_qa.py slice ==\n'
sed -n '1910,2005p' scripts/reborn_webui_v2_live_qa/run_live_qa.py
printf '\n== search to_contain_text usages ==\n'
rg -n "to_contain_text\(" scripts/reborn_webui_v2_live_qa/run_live_qa.py scripts -g '!**/node_modules/**' || true
printf '\n== relevant repo guidance ==\n'
rg -n "Playwright|to_contain_text|ignoreCase|ignore_case|pairing code|slack connect" -g 'CLAUDE.md' -g 'AGENTS.md' -g '.claude/**' -g '*.md' .Repository: nearai/ironclaw Length of output: 18420 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '\n== Playwright docs quick check ==\n'
python3 - <<'PY'
import urllib.request
url = "https://playwright.dev/python/docs/api/class-locatorassertions#locator-assertions-to-contain-text"
with urllib.request.urlopen(url, timeout=20) as r:
text = r.read().decode("utf-8", "ignore")
for needle in ["ignore_case", "ignoreCase", "Case-insensitive"]:
idx = text.find(needle)
print(f"{needle}: {idx}")
if idx != -1:
print(text[max(0, idx-200): idx+500])
PYRepository: nearai/ironclaw Length of output: 2782 🌐 Web query:
💡 Result: In Playwright Python, the Citations:
Align the Slack body check with the accepted instruction shapes. The helper allows either 🤖 Prompt for AI Agents |
||||||
|
|
||||||
|
|
||||||
| async def _slack_connect_case(ctx: LiveQaContext, *, case_name: str) -> ProbeResult: | ||||||
| from playwright.async_api import expect | ||||||
|
|
||||||
|
|
@@ -1978,10 +2034,10 @@ async def action(page: object) -> None: | |||||
| if not title: | ||||||
| raise AssertionError(f"Slack connect action title missing: {personal!r}") | ||||||
| instructions = str(action_body.get("instructions") or "") | ||||||
| if not _slack_connect_instructions_look_valid(instructions): | ||||||
| raise AssertionError(f"unexpected Slack connect instructions: {instructions!r}") | ||||||
| await expect(page.locator("body")).to_contain_text(title, timeout=15000) # type: ignore[attr-defined] | ||||||
| await expect(page.locator("body")).to_contain_text("pairing code", timeout=15000) # type: ignore[attr-defined] | ||||||
|
Contributor
There was a problem hiding this comment. The helper To ensure the stabilization is robust and backward-compatible as intended, use a regular expression that matches either instruction variant.
Suggested change
|
||||||
| observed["slack_display_name"] = personal.get("display_name") | ||||||
| observed["slack_connect_title"] = title | ||||||
| observed["slack_connect_instructions"] = instructions | ||||||
|
|
@@ -2952,6 +3008,7 @@ async def _routine_creation_case( | |||||
| required_text=required_text, | ||||||
| timeout=180.0, | ||||||
| extra_details=details, | ||||||
| routine_confirmation_follow_up=True, | ||||||
| ) | ||||||
| after_count = _trigger_record_count(ctx.reborn_home, count_name) | ||||||
| result.details["trigger_records_after"] = after_count | ||||||
|
|
@@ -3007,11 +3064,7 @@ async def _slack_delivery_routine_case( | |||||
| f"{routine_instruction} The routine's final answer and Slack message must " | ||||||
| f"include the exact marker {delivery_marker}. Create the routine now; do not " | ||||||
| "run it immediately. During routine creation, do not perform the routine's " | ||||||
| "live check, web/search/HTTP lookup, or Slack send. " | ||||||
| f"In your final answer include the exact marker {creation_marker} and include " | ||||||
| "the text routine." | ||||||
| ), | ||||||
|
|
@@ -3509,11 +3562,6 @@ async def case_qa_7c_slack_bug_logger_routine(ctx: LiveQaContext) -> ProbeResult | |||||
| required_text=["trigger|routine|automation|cron|schedule|fires|watches", "bug"], | ||||||
| prompt=_qa_sheet_prompt("qa_7c_slack_bug_logger_routine"), | ||||||
| extensions=[ | ||||||
| { | ||||||
| "package_id": "google-drive", | ||||||
| "display_name": "Google Drive", | ||||||
|
|
@@ -3543,16 +3591,11 @@ async def case_qa_7c_slack_bug_logger_routine(ctx: LiveQaContext) -> ProbeResult | |||||
|
|
||||||
|
|
||||||
| async def case_qa_7a_slack_product_channel_connect(ctx: LiveQaContext) -> ProbeResult: | ||||||
| started = time.monotonic() | ||||||
| case_name = "qa_7a_slack_product_channel_connect" | ||||||
| observed: dict[str, object] = { | ||||||
| "preflight": "Slack DM delivery target is configured before user-story workflow cases" | ||||||
| } | ||||||
| try: | ||||||
| slack = _slack_preflight(ctx) | ||||||
| delivery_channel_id = _slack_delivery_channel_id(ctx) | ||||||
|
|
@@ -3579,40 +3622,6 @@ async def case_qa_7a_slack_product_channel_connect(ctx: LiveQaContext) -> ProbeR | |||||
| "Slack live QA delivery target must be a DM to the user; " | ||||||
| f"got channel_id={delivery_channel_id!r}" | ||||||
| ) | ||||||
| return _result(case_name, True, started, observed) | ||||||
| except Exception as exc: | ||||||
| return _result( | ||||||
|
|
||||||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
📐 Maintainability & Code Quality | 🔵 Trivial | ⚡ Quick win
Guidance text duplicated by hand across two files.
This
search()message and thebuiltin.extension_searchmanifest description inextension_lifecycle_capabilities.rs(line 71) both encode the same "prefer the configured outbound delivery target over activating" rule in independently-worded, hand-maintained strings, only cross-checked via substring assertions in each file's tests. A future edit to one could silently drift from the other since nothing enforces they agree.Consider extracting the shared clause(s) into a
const/helper so both call sites compose from one source of truth.🤖 Prompt for AI Agents