diff --git a/.github/workflows/live-canary.yml b/.github/workflows/live-canary.yml index 11814d148d1..5ab91e70a73 100644 --- a/.github/workflows/live-canary.yml +++ b/.github/workflows/live-canary.yml @@ -495,12 +495,37 @@ jobs: retention-days: 14 reborn-webui-v2-live-qa: - name: Reborn WebUI v2 Live QA + name: Reborn WebUI v2 Live QA (${{ matrix.shard_name }}) if: > (github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *') || (github.event_name == 'workflow_dispatch' && (inputs.lane == 'all' || inputs.lane == 'reborn-webui-v2-live-qa')) runs-on: ubuntu-latest timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - shard_id: qa-2 + shard_name: QA 2 + cases: qa_2a_gmail_connect,qa_2b_calendar_connect,qa_2c_drive_connect,qa_2d_calendar_prep_live_chat,qa_2e_calendar_prep_email_routine,qa_2f_calendar_prep_email_delivery + - shard_id: qa-3 + shard_name: QA 3 + cases: qa_3a_slack_connect,qa_3b_endpoint_status_live_chat,qa_3c_endpoint_status_slack_routine,qa_3d_endpoint_status_slack_delivery + - shard_id: qa-4 + shard_name: QA 4 + cases: qa_4a_gmail_connect,qa_4b_github_connect,qa_4c_github_release_live_chat,qa_4d_github_release_slack_routine,qa_4e_github_release_email_delivery + - shard_id: qa-5 + shard_name: QA 5 + cases: qa_5a_slack_connect,qa_5b_drive_connect,qa_5c_strategy_doc_knowledge_base,qa_5d_slack_strategy_doc_answer + - shard_id: qa-6 + shard_name: QA 6 + cases: qa_6a_gmail_connect,qa_6b_sheets_connect,qa_6c_gmail_to_sheet_live_chat,qa_6d_gmail_to_sheet_routine,qa_6e_gmail_to_sheet_delivery + - shard_id: qa-7 + shard_name: QA 7 + cases: qa_7a_slack_product_channel_connect,qa_7b_sheets_connect,qa_7c_slack_bug_logger_routine,qa_7d_slack_bug_message_trigger,qa_7e_slack_bug_sheet_delivery + - shard_id: qa-8 + shard_name: QA 8 + cases: qa_8a_slack_connect,qa_8b_hn_keyword_live_chat,qa_8c_hn_keyword_slack_routine,qa_8d_hn_keyword_slack_delivery env: REBORN_WEBUI_V2_LIVE_QA_LLM_API_KEY_ENV: NEARAI_API_KEY REBORN_WEBUI_V2_LIVE_QA_LLM_PROVIDER_ID: nearai @@ -512,28 +537,102 @@ jobs: REBORN_WEBUI_V2_LIVE_QA_SLACK_TEAM_ID: ${{ vars.REBORN_WEBUI_V2_LIVE_QA_SLACK_TEAM_ID }} REBORN_WEBUI_V2_LIVE_QA_SLACK_API_APP_ID: ${{ vars.REBORN_WEBUI_V2_LIVE_QA_SLACK_API_APP_ID }} REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_CHANNEL_ID: ${{ vars.REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_CHANNEL_ID }} + REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_USER_ID: ${{ vars.REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_USER_ID }} REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_SUBJECT_USER_ID: ${{ vars.REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_SUBJECT_USER_ID }} LIVE_OPENAI_COMPATIBLE_BASE_URL: ${{ vars.LIVE_OPENAI_COMPATIBLE_BASE_URL || 'https://cloud-api.near.ai/v1' }} LIVE_OPENAI_COMPATIBLE_MODEL: ${{ vars.LIVE_OPENAI_COMPATIBLE_MODEL || 'Qwen/Qwen3.5-122B-A10B' }} steps: + - name: Resolve Reborn WebUI v2 live QA cases + id: resolve_reborn_webui_v2_cases + shell: bash + env: + REQUESTED_CASES: ${{ github.event_name == 'schedule' && 'all' || inputs.cases || vars.REBORN_WEBUI_V2_LIVE_QA_CASES || 'all' }} + SHARD_CASES: ${{ matrix.cases }} + ALL_SHARD_CASES: >- + qa_2a_gmail_connect,qa_2b_calendar_connect,qa_2c_drive_connect,qa_2d_calendar_prep_live_chat,qa_2e_calendar_prep_email_routine,qa_2f_calendar_prep_email_delivery, + qa_3a_slack_connect,qa_3b_endpoint_status_live_chat,qa_3c_endpoint_status_slack_routine,qa_3d_endpoint_status_slack_delivery, + qa_4a_gmail_connect,qa_4b_github_connect,qa_4c_github_release_live_chat,qa_4d_github_release_slack_routine,qa_4e_github_release_email_delivery, + qa_5a_slack_connect,qa_5b_drive_connect,qa_5c_strategy_doc_knowledge_base,qa_5d_slack_strategy_doc_answer, + qa_6a_gmail_connect,qa_6b_sheets_connect,qa_6c_gmail_to_sheet_live_chat,qa_6d_gmail_to_sheet_routine,qa_6e_gmail_to_sheet_delivery, + qa_7a_slack_product_channel_connect,qa_7b_sheets_connect,qa_7c_slack_bug_logger_routine,qa_7d_slack_bug_message_trigger,qa_7e_slack_bug_sheet_delivery, + qa_8a_slack_connect,qa_8b_hn_keyword_live_chat,qa_8c_hn_keyword_slack_routine,qa_8d_hn_keyword_slack_delivery + run: | + set -euo pipefail + cases="$(echo "${REQUESTED_CASES:-}" | xargs)" + if [[ -z "${cases}" || "${cases}" == "all" || "${cases}" == "ALL" || "${cases}" == "*" ]]; then + echo "CASES=${SHARD_CASES}" >> "${GITHUB_ENV}" + echo "REBORN_WEBUI_V2_LIVE_QA_CASES_RESOLUTION=matrix-shard" >> "${GITHUB_ENV}" + echo "skip_shard=0" >> "${GITHUB_OUTPUT}" + echo "CASES=${cases:-} requested; running shard cases: ${SHARD_CASES}" + else + selected=() + IFS=',' read -ra requested_cases <<< "${cases}" + IFS=',' read -ra shard_cases <<< "${SHARD_CASES}" + IFS=',' read -ra all_cases <<< "${ALL_SHARD_CASES}" + for requested in "${requested_cases[@]}"; do + requested="$(echo "${requested}" | xargs)" + [[ -n "${requested}" ]] || continue + known=0 + for known_case in "${all_cases[@]}"; do + known_case="$(echo "${known_case}" | xargs)" + if [[ "${requested}" == "${known_case}" ]]; then + known=1 + break + fi + done + if [[ ${known} -ne 1 ]]; then + echo "::error::Unknown Reborn WebUI v2 live QA case: ${requested}" + exit 1 + fi + for shard_case in "${shard_cases[@]}"; do + if [[ "${requested}" == "${shard_case}" ]]; then + selected+=("${requested}") + break + fi + done + done + if [[ ${#selected[@]} -eq 0 ]]; then + echo "REBORN_WEBUI_V2_LIVE_QA_SKIP_SHARD=1" >> "${GITHUB_ENV}" + echo "REBORN_WEBUI_V2_LIVE_QA_CASES_RESOLUTION=requested-not-in-shard" >> "${GITHUB_ENV}" + echo "skip_shard=1" >> "${GITHUB_OUTPUT}" + echo "Requested cases do not belong to this shard; skipping." + else + joined="$(IFS=,; echo "${selected[*]}")" + echo "CASES=${joined}" >> "${GITHUB_ENV}" + echo "REBORN_WEBUI_V2_LIVE_QA_CASES_RESOLUTION=requested-filtered" >> "${GITHUB_ENV}" + echo "skip_shard=0" >> "${GITHUB_OUTPUT}" + echo "Running requested cases in this shard: ${joined}" + fi + fi + - name: Reject unsafe Reborn WebUI v2 live QA target refs + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' && github.event_name == 'workflow_dispatch' && inputs.target_ref != '' + run: | + echo "::error::target_ref is disabled for reborn-webui-v2-live-qa because this lane runs with live secrets." + exit 1 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' with: - ref: ${{ inputs.target_ref || github.ref }} persist-credentials: false - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' with: targets: wasm32-wasip2 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' with: python-version: "3.12" - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' with: key: live-canary-reborn-webui-v2-live-qa - name: Install cargo-component + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' run: cargo install cargo-component --locked || true - name: Build WASM channels + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' run: ./scripts/build-wasm-extensions.sh --channels - name: Materialize Reborn WebUI v2 live QA sensitive secrets to files + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' shell: bash env: NEARAI_API_KEY: ${{ secrets.NEARAI_API_KEY || secrets.LIVE_OPENAI_COMPATIBLE_API_KEY }} @@ -573,37 +672,24 @@ jobs: write_secret "AUTH_LIVE_GITHUB_TOKEN" "${AUTH_LIVE_GITHUB_TOKEN:-}" write_secret "TELEGRAM_BOT_TOKEN" "${TELEGRAM_BOT_TOKEN:-}" write_secret "TELEGRAM_WEBHOOK_SECRET" "${TELEGRAM_WEBHOOK_SECRET:-}" - - name: Resolve Reborn WebUI v2 live QA cases - shell: bash - env: - REQUESTED_CASES: ${{ github.event_name == 'schedule' && 'all' || inputs.cases || vars.REBORN_WEBUI_V2_LIVE_QA_CASES || '' }} - run: | - set -euo pipefail - cases="$(echo "${REQUESTED_CASES:-}" | xargs)" - if [[ "${cases}" == "all" ]]; then - echo "CASES=all" >> "${GITHUB_ENV}" - echo "REBORN_WEBUI_V2_LIVE_QA_CASES_RESOLUTION=promoted-non-telegram" >> "${GITHUB_ENV}" - echo "CASES=all requested; running the promoted non-Telegram Reborn QA suite." - else - echo "CASES=${cases}" >> "${GITHUB_ENV}" - echo "REBORN_WEBUI_V2_LIVE_QA_CASES_RESOLUTION=requested" >> "${GITHUB_ENV}" - fi - name: Run Reborn WebUI v2 live QA lane + if: steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' env: LANE: reborn-webui-v2-live-qa - PROVIDER: reborn-webui-v2 + PROVIDER: reborn-webui-v2-${{ matrix.shard_id }} COMMAND_TIMEOUT: 90m PLAYWRIGHT_INSTALL: with-deps run: scripts/live-canary/run.sh - name: Scrub artifacts - if: always() + if: always() && steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' run: scripts/live-canary/scrub-artifacts.sh artifacts/live-canary - name: Upload artifacts - if: always() + if: always() && steps.resolve_reborn_webui_v2_cases.outputs.skip_shard != '1' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: live-canary-reborn-webui-v2-live-qa + name: live-canary-reborn-webui-v2-live-qa-${{ matrix.shard_id }} path: artifacts/live-canary/ + if-no-files-found: ignore retention-days: 14 persona-rotating: diff --git a/crates/ironclaw_first_party_extensions/assets/google-drive/manifest.toml b/crates/ironclaw_first_party_extensions/assets/google-drive/manifest.toml index f4e20b7091f..e46923ab203 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-drive/manifest.toml +++ b/crates/ironclaw_first_party_extensions/assets/google-drive/manifest.toml @@ -2,7 +2,7 @@ schema_version = "reborn.extension_manifest.v2" id = "google-drive" name = "Google Drive" version = "0.1.0" -description = "Google Drive capabilities for searching, accessing, uploading, sharing, and organizing files and folders." +description = "Google Drive capabilities for searching, accessing, uploading, sharing, and organizing files and folders, including finding Google Sheets/spreadsheets, Docs, and Slides by name or title." trust = "first_party_requested" [runtime] @@ -11,7 +11,7 @@ module = "wasm/google_drive_tool.wasm" [[capabilities]] id = "google-drive.list_files" -description = "Search or list files and folders." +description = "Search or list files and folders, including Google Sheets/spreadsheets by name or title." effects = ["dispatch_capability", "network", "use_secret"] runtime_credentials = [ { handle = "google_runtime_token", source = { type = "product_auth_account", provider = "google", setup = { kind = "oauth", scopes = ["https://www.googleapis.com/auth/drive.readonly"] } }, provider_scopes = ["https://www.googleapis.com/auth/drive.readonly"], audience = { scheme = "https", host_pattern = "www.googleapis.com" }, target = { type = "header", name = "authorization", prefix = "Bearer " } }, diff --git a/crates/ironclaw_first_party_extensions/assets/google-drive/prompts/google-drive/list_files.md b/crates/ironclaw_first_party_extensions/assets/google-drive/prompts/google-drive/list_files.md index f1166b356c7..d3223c90ff3 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-drive/prompts/google-drive/list_files.md +++ b/crates/ironclaw_first_party_extensions/assets/google-drive/prompts/google-drive/list_files.md @@ -1,3 +1,5 @@ Search or list files and folders. +Use this to find Google Workspace files by name/title, including Google Sheets. For example, to find a spreadsheet by exact title, query `name = '' and mimeType = 'application/vnd.google-apps.spreadsheet' and trashed = false`. + The host selects this operation from the capability id. Provide only the parameters described by the input schema; do not include an action field. diff --git a/crates/ironclaw_first_party_extensions/assets/google-drive/schemas/google-drive/list_files.input.v1.json b/crates/ironclaw_first_party_extensions/assets/google-drive/schemas/google-drive/list_files.input.v1.json index 3050794c2f9..2bd8c55256f 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-drive/schemas/google-drive/list_files.input.v1.json +++ b/crates/ironclaw_first_party_extensions/assets/google-drive/schemas/google-drive/list_files.input.v1.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google Drive list_files", - "description": "Search or list files and folders.", + "description": "Search or list files and folders, including Google Sheets/spreadsheets by name or title.", "type": "object", "required": [], "properties": { @@ -10,7 +10,7 @@ "string", "null" ], - "description": "Drive search query." + "description": "Drive search query. To find a Google Sheet by exact title, use a query like name = '' and mimeType = 'application/vnd.google-apps.spreadsheet' and trashed = false." }, "page_size": { "type": "integer", diff --git a/crates/ironclaw_first_party_extensions/assets/google-drive/wasm-src/src/lib.rs b/crates/ironclaw_first_party_extensions/assets/google-drive/wasm-src/src/lib.rs index 4dcf4977762..60045ca00b0 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-drive/wasm-src/src/lib.rs +++ b/crates/ironclaw_first_party_extensions/assets/google-drive/wasm-src/src/lib.rs @@ -1,7 +1,8 @@ //! Google Drive WASM Tool for IronClaw. //! //! Provides Google Drive integration for searching, accessing, uploading, -//! sharing, and organizing files and folders. Supports both personal and +//! sharing, and organizing files and folders, including finding Google Sheets/ +//! spreadsheets, Docs, and Slides by name/title. Supports both personal and //! shared (organizational) drives. //! //! # Capabilities Required @@ -28,6 +29,7 @@ //! //! ```json //! {"action": "list_files", "query": "name contains 'report' and mimeType = 'application/pdf'"} +//! {"action": "list_files", "query": "name = '' and mimeType = 'application/vnd.google-apps.spreadsheet' and trashed = false"} //! {"action": "list_files", "corpora": "drive", "drive_id": "0ABcd...", "query": "trashed = false"} //! {"action": "share_file", "file_id": "abc123", "email": "alice@company.com", "role": "writer"} //! ``` @@ -70,7 +72,8 @@ impl exports::near::agent::tool::Guest for GoogleDriveTool { fn description() -> String { "Google Drive integration for searching, accessing, uploading, sharing, and organizing \ - files and folders. Supports personal drives and shared (organizational) drives via the \ + files and folders, including finding Google Sheets/spreadsheets, Docs, and Slides by \ + name or title. Supports personal drives and shared (organizational) drives via the \ corpora parameter. Can search with Drive query syntax, download text files, upload new \ files, manage folder structure, and control sharing permissions. The host injects a \ Google product-auth credential with the Drive scope. \ diff --git a/crates/ironclaw_first_party_extensions/assets/google-sheets/manifest.toml b/crates/ironclaw_first_party_extensions/assets/google-sheets/manifest.toml index 001a58c5725..895977d0453 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-sheets/manifest.toml +++ b/crates/ironclaw_first_party_extensions/assets/google-sheets/manifest.toml @@ -2,7 +2,7 @@ schema_version = "reborn.extension_manifest.v2" id = "google-sheets" name = "Google Sheets" version = "0.1.0" -description = "Google Sheets capabilities for creating, reading, writing, and formatting spreadsheets." +description = "Google Sheets capabilities for creating, reading, writing, and formatting spreadsheets. Use Google Drive list_files to find existing spreadsheets by name/title." trust = "first_party_requested" [runtime] @@ -25,7 +25,7 @@ required_host_ports = ["host.runtime.http_egress"] [[capabilities]] id = "google-sheets.get_spreadsheet" -description = "Get spreadsheet metadata." +description = "Get spreadsheet metadata by spreadsheet ID. If the user only provided a spreadsheet name/title, search Google Drive first." effects = ["dispatch_capability", "network", "use_secret"] runtime_credentials = [ { handle = "google_runtime_token", source = { type = "product_auth_account", provider = "google", setup = { kind = "oauth", scopes = ["https://www.googleapis.com/auth/spreadsheets.readonly"] } }, provider_scopes = ["https://www.googleapis.com/auth/spreadsheets.readonly"], audience = { scheme = "https", host_pattern = "sheets.googleapis.com" }, target = { type = "header", name = "authorization", prefix = "Bearer " } }, @@ -39,7 +39,7 @@ required_host_ports = ["host.runtime.http_egress"] [[capabilities]] id = "google-sheets.read_values" -description = "Read cell values from a range." +description = "Read cell values from a range by spreadsheet ID. If the user only provided a spreadsheet name/title, search Google Drive first." effects = ["dispatch_capability", "network", "use_secret"] runtime_credentials = [ { handle = "google_runtime_token", source = { type = "product_auth_account", provider = "google", setup = { kind = "oauth", scopes = ["https://www.googleapis.com/auth/spreadsheets.readonly"] } }, provider_scopes = ["https://www.googleapis.com/auth/spreadsheets.readonly"], audience = { scheme = "https", host_pattern = "sheets.googleapis.com" }, target = { type = "header", name = "authorization", prefix = "Bearer " } }, @@ -81,7 +81,7 @@ required_host_ports = ["host.runtime.http_egress"] [[capabilities]] id = "google-sheets.append_values" -description = "Append rows after existing data." +description = "Append rows after existing data by spreadsheet ID. If the user only provided a spreadsheet name/title, search Google Drive first." effects = ["dispatch_capability", "network", "use_secret", "external_write"] runtime_credentials = [ { handle = "google_runtime_token", source = { type = "product_auth_account", provider = "google", setup = { kind = "oauth", scopes = ["https://www.googleapis.com/auth/spreadsheets"] } }, provider_scopes = ["https://www.googleapis.com/auth/spreadsheets"], audience = { scheme = "https", host_pattern = "sheets.googleapis.com" }, target = { type = "header", name = "authorization", prefix = "Bearer " } }, diff --git a/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/append_values.md b/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/append_values.md index a89c2d7add5..705fda1d711 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/append_values.md +++ b/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/append_values.md @@ -1,3 +1,5 @@ -Append rows after existing data. +Append rows after existing data by spreadsheet ID. + +If the user provided only a spreadsheet name/title, first use Google Drive `google-drive.list_files` to find the spreadsheet file ID. The host selects this operation from the capability id. Provide only the parameters described by the input schema; do not include an action field. diff --git a/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/get_spreadsheet.md b/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/get_spreadsheet.md index bfb9dcaf694..af3fb7c5ad3 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/get_spreadsheet.md +++ b/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/get_spreadsheet.md @@ -1,3 +1,5 @@ -Get spreadsheet metadata. +Get spreadsheet metadata by spreadsheet ID. + +If the user provided only a spreadsheet name/title, first use Google Drive `google-drive.list_files` to find the spreadsheet file ID. The host selects this operation from the capability id. Provide only the parameters described by the input schema; do not include an action field. diff --git a/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/read_values.md b/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/read_values.md index 80d360592cc..1e7178e9ea4 100644 --- a/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/read_values.md +++ b/crates/ironclaw_first_party_extensions/assets/google-sheets/prompts/google-sheets/read_values.md @@ -1,3 +1,5 @@ -Read cell values from a range. +Read cell values from a range by spreadsheet ID. + +If the user provided only a spreadsheet name/title, first use Google Drive `google-drive.list_files` to find the spreadsheet file ID. The host selects this operation from the capability id. Provide only the parameters described by the input schema; do not include an action field. diff --git a/crates/ironclaw_host_runtime/src/first_party_tools/memory.rs b/crates/ironclaw_host_runtime/src/first_party_tools/memory.rs index 5425091c188..9ec6f6a2468 100644 --- a/crates/ironclaw_host_runtime/src/first_party_tools/memory.rs +++ b/crates/ironclaw_host_runtime/src/first_party_tools/memory.rs @@ -30,6 +30,7 @@ pub const MEMORY_WRITE_CAPABILITY_ID: &str = "builtin.memory_write"; pub const MEMORY_READ_CAPABILITY_ID: &str = "builtin.memory_read"; pub const MEMORY_TREE_CAPABILITY_ID: &str = "builtin.memory_tree"; const MEMORY_PROMPT_SAFETY_EXTENSION_ID: &str = "memory.prompt_safety"; +const MEMORY_SEARCH_SCOPE: &str = "reborn_internal_persistent_memory"; struct MemoryServices { invocation: MemoryInvocation, @@ -62,7 +63,7 @@ pub(super) fn manifests() -> Result, ExtensionError> { Ok(vec![ first_party_capability_manifest( MEMORY_SEARCH_CAPABILITY_ID, - "Search Reborn persistent memory documents in the current tenant/user/agent/project scope", + "Search only Reborn internal persistent memory documents in the current tenant/user/agent/project scope. This does not search connected app or extension data.", vec![EffectKind::ReadFilesystem], PermissionMode::Allow, resource_profile(), @@ -447,6 +448,8 @@ fn search_response_to_value(response: MemoryServiceSearchResponse) -> Value { "query": response.query, "results": results, "result_count": result_count, + "search_scope": MEMORY_SEARCH_SCOPE, + "external_services_searched": false, }) } @@ -586,6 +589,11 @@ mod tests { .expect("memory_search should succeed through IronClaw memory facade"); assert_eq!(result.output["result_count"], 1); + assert_eq!( + result.output["search_scope"], + "reborn_internal_persistent_memory" + ); + assert_eq!(result.output["external_services_searched"], false); assert_eq!( result.output["results"][0]["content"], "captured through IronClaw memory" diff --git a/crates/ironclaw_host_runtime/src/first_party_tools/schemas.rs b/crates/ironclaw_host_runtime/src/first_party_tools/schemas.rs index 2596691539a..ee730155454 100644 --- a/crates/ironclaw_host_runtime/src/first_party_tools/schemas.rs +++ b/crates/ironclaw_host_runtime/src/first_party_tools/schemas.rs @@ -46,10 +46,11 @@ pub(crate) fn resolve_builtin_input_schema_ref(reference: &str) -> Option "schemas/builtin/http-save.input.v1.json" => http_schema(true), "schemas/builtin/memory_search.input.v1.json" => json!({ "type": "object", + "description": "Searches only Reborn internal persistent memory. This does not search connected app or extension data.", "properties": { "query": { "type": "string", - "description": "Preferred natural language search query for persistent memory" + "description": "Preferred natural language search query for Reborn internal persistent memory" }, "q": { "type": "string", diff --git a/crates/ironclaw_host_runtime/tests/first_party_builtin_tools.rs b/crates/ironclaw_host_runtime/tests/first_party_builtin_tools.rs index 2415e9a7fd6..70bac9be0e4 100644 --- a/crates/ironclaw_host_runtime/tests/first_party_builtin_tools.rs +++ b/crates/ironclaw_host_runtime/tests/first_party_builtin_tools.rs @@ -320,6 +320,67 @@ async fn builtin_first_party_surface_lists_allowed_tools_in_registry_order() { assert!(!properties.contains_key("run_in_background")); } +#[tokio::test] +async fn builtin_memory_search_surface_declares_internal_scope_boundary() { + let runtime = runtime(); + let request = VisibleCapabilityRequest::new( + execution_context(all_builtin_capability_ids()), + SurfaceKind::new("agent_loop").unwrap(), + ) + .with_policy(CapabilitySurfacePolicy::allow_all()) + .with_provider_trust(provider_trust()); + + let surface = runtime.visible_capabilities(request).await.unwrap(); + + let memory_search = surface + .capabilities + .iter() + .find(|capability| capability.descriptor.id.as_str() == MEMORY_SEARCH_CAPABILITY_ID) + .expect("memory_search must appear in surface"); + assert!( + memory_search + .descriptor + .description + .contains("only Reborn internal persistent memory"), + "memory_search description should declare the internal-memory boundary: {}", + memory_search.descriptor.description + ); + assert!( + memory_search + .descriptor + .description + .contains("does not search connected app or extension data"), + "memory_search description should avoid implying it can search external apps: {}", + memory_search.descriptor.description + ); + + let schema = &memory_search.descriptor.parameters_schema; + let schema_description = schema + .get("description") + .and_then(Value::as_str) + .expect("memory_search schema should describe its search scope"); + assert!( + schema_description.contains("only Reborn internal persistent memory"), + "memory_search schema description should declare scope: {schema_description}" + ); + assert!( + schema_description.contains("does not search connected app or extension data"), + "memory_search schema description should avoid external-app ambiguity: {schema_description}" + ); + + let query_description = schema + .get("properties") + .and_then(Value::as_object) + .and_then(|properties| properties.get("query")) + .and_then(|query| query.get("description")) + .and_then(Value::as_str) + .expect("memory_search query should have a description"); + assert!( + query_description.contains("Reborn internal persistent memory"), + "memory_search query description should declare internal memory: {query_description}" + ); +} + #[tokio::test] async fn builtin_trigger_create_input_schema_declares_schedule_one_of() { let runtime = runtime_with_trigger_repository(Arc::new(InMemoryTriggerRepository::default())); diff --git a/crates/ironclaw_reborn_composition/src/available_extensions.rs b/crates/ironclaw_reborn_composition/src/available_extensions.rs index 3d369f85207..cbe0961f4e3 100644 --- a/crates/ironclaw_reborn_composition/src/available_extensions.rs +++ b/crates/ironclaw_reborn_composition/src/available_extensions.rs @@ -1732,6 +1732,27 @@ mod tests { } } + #[test] + fn bundled_google_sheet_queries_discover_drive_lookup_tool() { + let catalog = AvailableExtensionCatalog::from_first_party_assets().unwrap(); + + for query in ["google sheets", "google sheet", "spreadsheet"] { + let ids = catalog + .search(query) + .map(|package| package.package_ref.id.as_str()) + .collect::>(); + + assert!( + ids.contains("google-drive"), + "{query} should discover Google Drive for spreadsheet-name lookup; got {ids:?}" + ); + assert!( + ids.contains("google-sheets"), + "{query} should still discover Google Sheets; got {ids:?}" + ); + } + } + #[test] fn bundled_github_read_only_capabilities_default_allow_without_relaxing_writes() { let catalog = AvailableExtensionCatalog::from_first_party_assets().unwrap(); diff --git a/scripts/live-canary/ACCOUNTS.md b/scripts/live-canary/ACCOUNTS.md index 2aee1cf3d7d..63de6f2a13f 100644 --- a/scripts/live-canary/ACCOUNTS.md +++ b/scripts/live-canary/ACCOUNTS.md @@ -148,6 +148,9 @@ for WebUI Slack workflow coverage. - `IRONCLAW_REBORN_SLACK_BOT_TOKEN` - `REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_CHANNEL_ID` when a stable delivery target should be pinned instead of discovered from `auth.test` +- `REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_USER_ID` when the route channel is not + pinned and the discovery DM should target a specific QA Slack user; otherwise + discovery falls back to a Slackbot DM, never a public channel Telegram workflow cases require a real test bot token: diff --git a/scripts/live-canary/notify_slack.py b/scripts/live-canary/notify_slack.py index cdc087cc619..4431daca4ec 100755 --- a/scripts/live-canary/notify_slack.py +++ b/scripts/live-canary/notify_slack.py @@ -238,10 +238,10 @@ def _reborn_failure_message(entry: dict) -> str: for key in ("error", "message", "reason", "gate"): value = details.get(key) if value: - return _trim_slack_text(value) + return _trim_slack_text(value, 360) blocked = details.get("blocked") if blocked: - return _trim_slack_text(f"blocked={blocked}") + return _trim_slack_text(f"blocked={blocked}", 360) return "" @@ -608,7 +608,18 @@ def _format_reborn_tool_summary(cases: list[RebornQaCaseReport]) -> list[str]: return lines -def _format_reborn_qa_group(group: str, cases: list[RebornQaCaseReport]) -> dict: +def _format_reborn_failure_lines(cases: list[RebornQaCaseReport]) -> list[str]: + lines: list[str] = [] + for case in cases: + if case.success: + continue + rows = _qa_case_rows(case) + message = case.message or "failed" + lines.append(f"*Failure `{rows}`:* {message}") + return lines + + +def _format_reborn_qa_group(group: str, cases: list[RebornQaCaseReport]) -> list[dict]: passed = sum(1 for case in cases if case.success) failed = len(cases) - passed status = ":white_check_mark:" if failed == 0 else ":x:" @@ -625,31 +636,47 @@ def _format_reborn_qa_group(group: str, cases: list[RebornQaCaseReport]) -> dict f"{status} *QA {group}* — {passed}/{len(cases)} passed{duration}", f"*Cases:* {_trim_slack_text('; '.join(case_summaries), 900)}", ] - failed_messages = [ - f"`{_qa_case_rows(case)}` {case.message or 'failed'}" - for case in cases - if not case.success - ] - if failed_messages: - lines.append(f"*Failures:* {_trim_slack_text('; '.join(failed_messages), 700)}") + lines.extend(_format_reborn_failure_lines(cases)) lines.extend(_format_reborn_tool_summary(cases)) - return { - "type": "section", - "text": { - "type": "mrkdwn", - "text": _trim_slack_block_text("\n".join(lines), 2900), - }, - } + blocks: list[dict] = [] + current: list[str] = [] + continuation_header = f"{status} *QA {group}* — continued" + for line in lines: + candidate = "\n".join([*current, line]) + if current and len(candidate) > 2900: + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": _trim_slack_block_text("\n".join(current), 2900), + }, + } + ) + current = [continuation_header, line] + else: + current.append(line) + if current: + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": _trim_slack_block_text("\n".join(current), 2900), + }, + } + ) + return blocks def _format_reborn_qa_groups(cases: list[RebornQaCaseReport]) -> list[dict]: grouped: dict[str, list[RebornQaCaseReport]] = {} for case in cases: grouped.setdefault(_qa_group_key(case), []).append(case) - return [ - _format_reborn_qa_group(group, grouped[group]) - for group in sorted(grouped, key=_qa_group_sort_key) - ] + blocks: list[dict] = [] + for group in sorted(grouped, key=_qa_group_sort_key): + blocks.extend(_format_reborn_qa_group(group, grouped[group])) + return blocks def slack_payload( @@ -667,6 +694,7 @@ def slack_payload( {"type": "header", "text": {"type": "plain_text", "text": header}}, ] for r in reports: + renders_reborn_qa_groups = bool(r.reborn_qa_cases) header_line = ( f"{emoji.get(r.status, ':grey_question:')} *{r.lane}* ({r.provider}) — " f"{r.passed}/{r.tests} passed, {r.failed} failed in {r.duration_s:.0f}s" @@ -676,7 +704,11 @@ def slack_payload( # fields. The shape mirrors the issue-friendly format reviewers # asked for so a Slack reader can paste it straight into a # GitHub issue if needed. - if r.status == "fail" and (r.test_name or r.error or r.root_cause): + if ( + r.status == "fail" + and not renders_reborn_qa_groups + and (r.test_name or r.error or r.root_cause) + ): if r.test_name: lines.append(f" *Test:* `{r.test_name}`") if r.error: @@ -685,16 +717,16 @@ def slack_payload( lines.append(f" *Root Cause:* {r.root_cause}") if r.fix: lines.append(f" *Fix:* {r.fix}") - elif r.reason: + elif r.reason and not renders_reborn_qa_groups: # For passing/skipped lanes we keep the existing single- # line reason summary (Haiku's free-form notable). lines.append(f"> {r.reason}") - if r.tools_used: + if r.tools_used and not renders_reborn_qa_groups: lines.append(f"tools: {', '.join(r.tools_used)} (≈{r.tool_calls_total} calls)") - if r.notable: + if r.notable and not renders_reborn_qa_groups: lines.append(f"_{r.notable}_") blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": "\n".join(lines)}}) - if r.reborn_qa_cases: + if renders_reborn_qa_groups: remaining = max(0, SLACK_MAX_BLOCKS - len(blocks)) blocks.extend(_format_reborn_qa_groups(r.reborn_qa_cases)[:remaining]) diff --git a/scripts/live-canary/test_notify_slack.py b/scripts/live-canary/test_notify_slack.py index a25999a29d4..94230603856 100644 --- a/scripts/live-canary/test_notify_slack.py +++ b/scripts/live-canary/test_notify_slack.py @@ -278,8 +278,8 @@ def test_slack_payload_renders_each_reborn_qa_row(self): lane="reborn-webui-v2-live-qa", provider="reborn-webui-v2", passed=1, - failed=1, - tests=2, + failed=2, + tests=3, duration_s=1.2, status="fail", reborn_qa_cases=[ @@ -312,6 +312,17 @@ def test_slack_payload_renders_each_reborn_qa_row(self): ) ], ), + notify.RebornQaCaseReport( + rows=("2E",), + case="qa_2e_calendar_prep_email_routine", + feature="Scheduled meeting-prep email routine", + success=False, + latency_ms=0, + message=( + "assistant returned success but routine scope " + "'reborn-qa-2e-calendar-prep-email' did not add a trigger_record" + ), + ), ], ) @@ -326,19 +337,27 @@ def test_slack_payload_renders_each_reborn_qa_row(self): self.assertEqual(len(qa_sections), 1) self.assertTrue( any( - "*reborn-webui-v2-live-qa* (reborn-webui-v2) — 1/2 passed" + "*reborn-webui-v2-live-qa* (reborn-webui-v2) — 1/3 passed" in text for text in section_texts ) ) qa_text = qa_sections[0] - self.assertIn("1/2 passed", qa_text) + self.assertIn("1/3 passed", qa_text) self.assertIn("\n*Cases:*", qa_text) self.assertIn("\n*Tools:*", qa_text) self.assertNotIn("\n*Tool I/O digests:*", qa_text) self.assertIn("`2A` Gmail connection flow", qa_text) self.assertIn("`2D` Calendar prep assistant using Google Docs and live news", qa_text) - self.assertIn("requires live Google runtime access", qa_text) + self.assertIn( + "*Failure `2D`:* requires live Google runtime access", + qa_text, + ) + self.assertIn( + "*Failure `2E`:* assistant returned success but routine scope " + "'reborn-qa-2e-calendar-prep-email' did not add a trigger_record", + qa_text, + ) self.assertIn("*Tools:* 2 calls across 2 tools", qa_text) self.assertNotIn("in#1234567890", qa_text) self.assertNotIn("out#9876543210", qa_text) @@ -396,6 +415,32 @@ def test_reborn_rows_fit_with_scheduled_all_lane_report(self): self.assertTrue(any("*QA 8*" in text for text in section_texts)) self.assertFalse(any("*QA 2A" in text for text in section_texts)) + def test_reborn_group_continuation_blocks_repeat_group_label(self): + cases = [ + notify.RebornQaCaseReport( + rows=("7A",), + case=f"qa_7a_failure_{idx}", + feature=f"Slack product channel connect {idx}", + success=False, + message="failure detail " + ("x" * 900), + ) + for idx in range(8) + ] + + blocks = notify._format_reborn_qa_group("7", cases) + section_texts = [ + block["text"]["text"] + for block in blocks + if block.get("type") == "section" + ] + + self.assertGreater(len(section_texts), 1) + self.assertTrue(section_texts[0].startswith(":x: *QA 7* — ")) + self.assertTrue( + all(text.startswith(":x: *QA 7* — ") for text in section_texts[1:]) + ) + self.assertTrue(any("continued" in text for text in section_texts[1:])) + if __name__ == "__main__": unittest.main() diff --git a/scripts/reborn_webui_v2_live_qa/case_matrix.py b/scripts/reborn_webui_v2_live_qa/case_matrix.py index d2a196094e2..8e11237753e 100644 --- a/scripts/reborn_webui_v2_live_qa/case_matrix.py +++ b/scripts/reborn_webui_v2_live_qa/case_matrix.py @@ -178,9 +178,10 @@ def qa_row_sort_key(row_id: str) -> tuple[int, str]: "rows": ["6C"], "feature": "CRM inbound email extraction to Google Sheet", "gate": ( - "requires a live Google OAuth account authorized for Gmail and Google " - "Sheets runtime execution plus test data, and Google OAuth refresh " - "env when the copied access token is expired" + "requires a live Google OAuth account authorized for Gmail, Google " + "Drive name lookup, and Google Sheets runtime execution plus test " + "data, and Google OAuth refresh env when the copied access token is " + "expired" ), }, "qa_6d_gmail_to_sheet_routine": { diff --git a/scripts/reborn_webui_v2_live_qa/google_api_helpers.py b/scripts/reborn_webui_v2_live_qa/google_api_helpers.py index 033941c173d..320a7db290b 100644 --- a/scripts/reborn_webui_v2_live_qa/google_api_helpers.py +++ b/scripts/reborn_webui_v2_live_qa/google_api_helpers.py @@ -101,6 +101,82 @@ async def _google_drive_file_id_by_name( return None +async def _create_google_spreadsheet_fixture( + *, + access_token: str, + title: str, + values: list[list[str]], + sheet_name: str = "Sheet1", +) -> dict[str, object]: + import httpx + + async with httpx.AsyncClient(timeout=30.0) as client: + create_response = await client.post( + "https://sheets.googleapis.com/v4/spreadsheets", + headers={"Authorization": f"Bearer {access_token}"}, + params={"fields": "spreadsheetId,spreadsheetUrl"}, + json={ + "properties": {"title": title}, + "sheets": [{"properties": {"title": sheet_name}}], + }, + ) + try: + create_payload: object = create_response.json() + except ValueError: + create_payload = {} + if create_response.status_code < 200 or create_response.status_code >= 300: + error = create_payload.get("error") if isinstance(create_payload, dict) else None + message = ( + error.get("message") if isinstance(error, dict) else str(create_payload)[:300] + ) + raise AssertionError( + "Google Sheets fixture create returned HTTP " + f"{create_response.status_code}: {message}" + ) + if not isinstance(create_payload, dict): + raise AssertionError(f"Google Sheets fixture create returned {create_payload!r}") + spreadsheet_id = str(create_payload.get("spreadsheetId") or "").strip() + if not spreadsheet_id: + raise AssertionError( + f"Google Sheets fixture create omitted spreadsheetId: {create_payload!r}" + ) + + values_written = False + if values: + update_response = await client.put( + "https://sheets.googleapis.com/v4/spreadsheets/" + f"{spreadsheet_id}/values/" + f"{urllib.parse.quote(f'{sheet_name}!A1', safe='!:$')}", + headers={"Authorization": f"Bearer {access_token}"}, + params={"valueInputOption": "RAW"}, + json={"majorDimension": "ROWS", "values": values}, + ) + try: + update_payload: object = update_response.json() + except ValueError: + update_payload = {} + if update_response.status_code < 200 or update_response.status_code >= 300: + error = update_payload.get("error") if isinstance(update_payload, dict) else None + message = ( + error.get("message") + if isinstance(error, dict) + else str(update_payload)[:300] + ) + raise AssertionError( + "Google Sheets fixture value update returned HTTP " + f"{update_response.status_code}: {message}" + ) + values_written = True + + return { + "spreadsheet_id": spreadsheet_id, + "spreadsheet_url": str(create_payload.get("spreadsheetUrl") or ""), + "title": title, + "sheet_name": sheet_name, + "values_written": values_written, + } + + async def _google_sheet_contains_marker( *, access_token: str, diff --git a/scripts/reborn_webui_v2_live_qa/run_live_qa.py b/scripts/reborn_webui_v2_live_qa/run_live_qa.py index 0a3d2f1243c..0dca265a048 100644 --- a/scripts/reborn_webui_v2_live_qa/run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/run_live_qa.py @@ -65,6 +65,7 @@ _section_env_name, ) from scripts.reborn_webui_v2_live_qa.google_api_helpers import ( # noqa: E402 + _create_google_spreadsheet_fixture, _extract_google_document_id, _extract_google_spreadsheet_id, _gmail_delivery_target_email, @@ -108,10 +109,75 @@ _slack_team_id_from_bot_token_env, ) +QA_SHEET_PROMPTS: dict[str, str] = { + "qa_2a_gmail_connect": """In WebUI, ask IronClaw “connect to Gmail.” Go through the auth flow. +Expected result: Gmail is connected""", + "qa_2b_calendar_connect": """In WebUI, ask IronClaw “connect to Google Calendar.” Go through the auth flow. +Expected result: Google Calendar is connected""", + "qa_2c_drive_connect": """In WebUI, ask IronClaw “connect to Google Drive.” Go through the auth flow. +Expected result: Google Drive is connected""", + "qa_2d_calendar_prep_live_chat": """In WebUI, ask IronClaw “For my next meeting, find information about the company that I am meeting with from my Google Docs and find the latest news.” +Expected result: Reference a Google Doc and the latest news""", + "qa_2e_calendar_prep_email_routine": """In WebUI, ask IronClaw, “Every 30 minutes, send me an email with a summary for my next meeting, including info about the company I will meet, based on the Google Drive docs and the latest news.” +Expected results: Routine created""", + "qa_3a_slack_connect": """In WebUI, ask IronClaw "connect to Slack." Go through the flow. +Expected result: Slack is connected""", + "qa_3b_endpoint_status_live_chat": """In WebUI, ask IronClaw "check if near.ai returns a 200 status." +Expected result: IronClaw reports the endpoint's current HTTP status""", + "qa_3c_endpoint_status_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, ping [endpoint URL] checking if it returns a 200 status and send result in a DM in slack" +Expected result: Routine created""", + "qa_4a_gmail_connect": """In WebUI, ask IronClaw "connect to Gmail." Go through the flow w/ Gmail. +Expected result: Gmail is connected""", + "qa_4b_github_connect": """In WebUI, ask IronClaw "connect to GitHub." Go through the auth flow. +Expected result: GitHub is connected""", + "qa_4c_github_release_live_chat": """In WebUI, ask IronClaw "summarize the latest release from https://github.com/nearai/ironclaw." +Expected result: summary of the most recent release""", + "qa_4d_github_release_slack_routine": """In WebUI, ask IronClaw, "Every 5 minutes, check https://github.com/nearai/ironclaw for latest releases and send me a Slack message summarizing any new ones." +Expected result: Routine created""", + "qa_5a_slack_connect": """In WebUI, ask IronClaw "connect to Slack." Go through the auth flow. +Expected result: Slack is connected""", + "qa_5b_drive_connect": """In WebUI, ask IronClaw "connect to Google Drive." Go through the auth flow. +Expected result: Google Drive is connected""", + "qa_5c_strategy_doc_knowledge_base": """In WebUI, ask IronClaw "use the NEAR AI Strategy doc in my Google Drive as your knowledge base for answering strategy questions." +Expected result: IronClaw references the doc and confirms it can answer from it""", + "qa_5d_slack_strategy_doc_answer": """In Slack, in a DM with IronClawm, ask a detailed strategy question about a Google doc (providing a link) +Expected result: Slack reply that answers the question, grounded in the strategy doc""", + "qa_6a_gmail_connect": """In WebUI, ask IronClaw "connect to Gmail." Go through the auth flow. +Expected result: Gmail is connected""", + "qa_6b_sheets_connect": """In WebUI, ask IronClaw "connect to Google Sheets." Go through the auth flow. +Expected result: Google Sheets is connected""", + "qa_6c_gmail_to_sheet_live_chat": """In WebUI, ask IronClaw "check my recent emails and add any from a near.ai address to my Google Sheet called ABC." +Expected result: ABC sheet has new rows for each near.ai inbound email""", + "qa_6d_gmail_to_sheet_routine": """In WebUI, ask IronClaw, "Every 30 minutes, check my inbox and add any new emails from a near.ai address to my Google Sheet called ABC." +Expected result: Routine created""", + "qa_7a_slack_product_channel_connect": """In WebUI, ask IronClaw "connect to Slack, using channel #product." Go through the flow +Expected result: Slack channel is connected""", + "qa_7b_sheets_connect": """In WebUI, ask IronClaw "connect to Google Sheets." Go through the auth flow. +Expected result: Google Sheets is connected""", + "qa_7c_slack_bug_logger_routine": """In WebUI, ask IronClaw "whenever I send a slack message starting with 'bug:', add it as a row to my bug logging Google Sheet." +Expected result: Routine/trigger created""", + "qa_7d_slack_bug_message_trigger": """In Slack, send a message starting with "bug:" +Expected result: Routine created""", + "qa_8a_slack_connect": """In WebUI, ask IronClaw "connect to Slack." Go through the auth flow. +Expected result: Slack is connected""", + "qa_8b_hn_keyword_live_chat": """In WebUI, ask IronClaw "search Hacker News for any recent posts mentioning 'IronClaw' or 'NEAR AI'." +Expected result: IronClaw reports any matching HN posts""", + "qa_8c_hn_keyword_slack_routine": """In WebUI, ask IronClaw, "Every hour, check Hacker News for new posts mentioning 'IronClaw' or 'NEAR AI' and send a summary to Slack." +Expected result: Routine created""", +} + + +def _qa_sheet_prompt(case_name: str) -> str: + try: + return QA_SHEET_PROMPTS[case_name] + except KeyError as exc: + raise AssertionError(f"QA sheet prompt is not hardcoded for {case_name}") from exc + DEFAULT_OUTPUT_DIR = ROOT / "artifacts" / "reborn-webui-v2-live-qa" DEFAULT_REBORN_HOME = Path("/tmp/ironclaw-reborn-real-slack") AUTH_TOKEN = "reborn-webui-v2-live-qa-token-0123456789abcdef" DEFAULT_USER_ID = "reborn-webui-v2-live-qa-user" +ENDPOINT_STATUS_URL = "https://near.ai" PROVIDER = "reborn-webui-v2" MODE = "live" HN_KEYWORD_SEARCH_URL = ( @@ -121,6 +187,13 @@ EXTENSION_SEARCH_CAPABILITY_ID = "builtin.extension_search" EXTENSION_INSTALL_CAPABILITY_ID = "builtin.extension_install" EXTENSION_ACTIVATE_CAPABILITY_ID = "builtin.extension_activate" +OUTBOUND_DELIVERY_TARGETS_LIST_CAPABILITY_ID = "builtin.outbound_delivery_targets_list" +QA_7A_CHAT_CONNECT_CAPABILITY_IDS = [ + EXTENSION_SEARCH_CAPABILITY_ID, + EXTENSION_INSTALL_CAPABILITY_ID, + EXTENSION_ACTIVATE_CAPABILITY_ID, +] +QA_7C_BUG_LOGGING_SHEET_TITLE = "bug logging Google Sheet" class LiveQaContext: @@ -741,7 +814,7 @@ async def _live_chat_case( *, case_name: str, prompt: str, - marker: str, + marker: str | None, required_text: list[str], timeout: float = 120.0, extra_details: dict[str, object] | None = None, @@ -800,6 +873,7 @@ async def action(page: object) -> None: True, started, { + "prompt": prompt, "marker": marker, "required_text": required_text, **(extra_details or {}), @@ -813,6 +887,7 @@ async def action(page: object) -> None: started, { "error": str(exc), + "prompt": prompt, "marker": marker, "required_text": required_text, **(extra_details or {}), @@ -826,7 +901,7 @@ async def _live_chat_with_extensions_case( *, case_name: str, prompt: str, - marker: str, + marker: str | None, required_text: list[str], extensions: list[dict[str, object]], timeout: float = 240.0, @@ -838,6 +913,7 @@ async def _live_chat_with_extensions_case( started = time.monotonic() observed: dict[str, object] = { "marker": marker, + "prompt": prompt, "required_text": required_text, "extensions": [extension["package_id"] for extension in extensions], **(extra_details or {}), @@ -917,7 +993,7 @@ async def _dismiss_visible_connect_action(page: object) -> bool: async def _wait_for_assistant_reply( page: object, *, - marker: str, + marker: str | None, required_text: list[str], timeout: float, ) -> str: @@ -934,7 +1010,8 @@ async def _wait_for_assistant_reply( if text: last_text = text normalized = text.lower() - if marker in text and all(piece.lower() in normalized for piece in required_text): + marker_matches = not marker or marker in text + if marker_matches and all(piece.lower() in normalized for piece in required_text): return text[-2000:] await asyncio.sleep(0.5) main_text = "" @@ -1084,33 +1161,30 @@ async def _wait_for_google_sheet_marker_after_slack_event( async def case_qa_3b_endpoint_status_live_chat(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_3B_ENDPOINT_STATUS_DONE" - url = "https://cloud-api.near.ai" + url = ENDPOINT_STATUS_URL live_status = await _live_http_status(url) return await _live_chat_case( ctx, case_name="qa_3b_endpoint_status_live_chat", - prompt=( - f"QA case 3B: check the current HTTP status for {url}. Use live HTTP " - "or web capabilities if available. If the endpoint does not return 200, " - "report the actual status code. In the final answer include the exact " - f"marker {marker} and include the text status." - ), - marker=marker, - required_text=["status", str(live_status)], + prompt=_qa_sheet_prompt("qa_3b_endpoint_status_live_chat"), + marker=None, + required_text=["status"], extra_details={"endpoint_url": url, "expected_status_code": live_status}, ) -def _trigger_record_count(reborn_home: Path, routine_name: str) -> int: +def _trigger_record_count(reborn_home: Path, routine_name: str | None = None) -> int: db_path = reborn_home / "local-dev" / "reborn-local-dev.db" if not db_path.exists(): return 0 with sqlite3.connect(db_path) as db: - cursor = db.execute( - "SELECT COUNT(*) FROM trigger_records WHERE name = ?", - (routine_name,), - ) + if routine_name: + cursor = db.execute( + "SELECT COUNT(*) FROM trigger_records WHERE name = ?", + (routine_name,), + ) + else: + cursor = db.execute("SELECT COUNT(*) FROM trigger_records") value = cursor.fetchone()[0] return int(value) @@ -1286,6 +1360,24 @@ async def _approve_slack_event_gates( } +async def _wait_for_slack_event_run_id( + ctx: LiveQaContext, + *, + event_id: str, + timeout: float = 180.0, +) -> str: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + run_id = _slack_event_run_id_for_event(ctx.reborn_home, event_id) + if run_id: + return run_id + await asyncio.sleep(1.0) + raise AssertionError( + "Slack event was not accepted into a Reborn run before timeout. " + f"event_id={event_id!r}" + ) + + async def _resolve_webui_approval_gate( ctx: LiveQaContext, *, @@ -1372,6 +1464,10 @@ def _slack_delivery_channel_id(ctx: LiveQaContext) -> str | None: return match.group(2) if match else None +def _slack_delivery_target_is_dm(channel_id: str | None) -> bool: + return bool(channel_id and channel_id.startswith("D")) + + async def _slack_history_contains_marker( ctx: LiveQaContext, *, @@ -1529,7 +1625,7 @@ async def _slack_connect_case(ctx: LiveQaContext, *, case_name: str) -> ProbeRes from playwright.async_api import expect started = time.monotonic() - prompt = "connect my Slack account" + prompt = _qa_sheet_prompt(case_name) observed: dict[str, object] = {"chat_connect_prompt": prompt} async def action(page: object) -> None: @@ -1645,6 +1741,15 @@ def _capability_run_statuses( return statuses +def _completed_capability_counts( + statuses: dict[str, list[str]], +) -> dict[str, int]: + return { + capability_id: capability_statuses.count("completed") + for capability_id, capability_statuses in statuses.items() + } + + async def _extension_chat_connect_case( ctx: LiveQaContext, *, @@ -1662,30 +1767,40 @@ async def _extension_chat_connect_case( EXTENSION_INSTALL_CAPABILITY_ID, EXTENSION_ACTIVATE_CAPABILITY_ID, ] - expected_capabilities = [*setup_capabilities, *verification_capabilities] - prompt = ( - f"QA connect case {case_name}: connect my {display_name} from this chat. " - f"Use extension_search for `{package_id}`, then install and activate " - f"`{package_id}` if it is not already active. {verification_instruction} " - "Do not create, update, send, or delete anything. In the final answer " - f"include the exact marker {marker} and include the words " - f"{display_name} connected." - ) + prompt = QA_SHEET_PROMPTS.get(case_name) + sheet_prompt = prompt is not None + expected_capabilities = ( + setup_capabilities + if sheet_prompt + else [*setup_capabilities, *verification_capabilities] + ) + marker_to_wait_for: str | None = None + if prompt is None: + prompt = ( + f"QA connect case {case_name}: connect my {display_name} from this chat. " + f"Use extension_search for `{package_id}`, then install and activate " + f"`{package_id}` if it is not already active. {verification_instruction} " + "Do not create, update, send, or delete anything. In the final answer " + f"include the exact marker {marker} and include the words " + f"{display_name} connected." + ) + marker_to_wait_for = marker chat = await _live_chat_case( ctx, case_name=case_name, prompt=prompt, - marker=marker, + marker=marker_to_wait_for, required_text=[display_name, "connected"], timeout=240.0, extra_details={ "chat_connect_flow": True, "package_id": package_id, "required_capabilities": expected_capabilities, + "verification_capabilities": verification_capabilities, + "verification_capabilities_required": not sheet_prompt, }, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "can't connect", "cannot connect", @@ -1706,6 +1821,8 @@ async def _extension_chat_connect_case( "display_name": display_name, "required_tools": required_tools, "required_capabilities": expected_capabilities, + "verification_capabilities": verification_capabilities, + "verification_capabilities_required": not sheet_prompt, } try: statuses = _capability_run_statuses(ctx.reborn_home, expected_capabilities) @@ -1869,12 +1986,11 @@ async def case_qa_2c_drive_connect(ctx: LiveQaContext) -> ProbeResult: async def case_qa_2d_calendar_prep_live_chat(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_2D_CALENDAR_PREP_DONE" return await _live_chat_with_extensions_case( ctx, case_name="qa_2d_calendar_prep_live_chat", - marker=marker, - required_text=["Calendar", "news"], + marker=None, + required_text=["Google", "news"], extensions=[ { "package_id": "google-calendar", @@ -1897,39 +2013,20 @@ async def case_qa_2d_calendar_prep_live_chat(ctx: LiveQaContext) -> ProbeResult: "required_tools": ["web-access.search"], }, ], - prompt=( - "QA case 2D: act as a meeting prep assistant. Use my live Google " - "Calendar connection to inspect upcoming events, and use live web " - "search for current NEAR AI news that could be useful context. If " - "there are no upcoming events, say that directly. Do not create, " - "update, or delete calendar events. In the final answer include the " - f"exact marker {marker}, include the word Calendar, and include the " - "word news." - ), + prompt=_qa_sheet_prompt("qa_2d_calendar_prep_live_chat"), timeout=300.0, ) async def case_qa_2e_calendar_prep_email_routine(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_2E_CALENDAR_EMAIL_ROUTINE_DONE" routine_name = "reborn-qa-2e-calendar-prep-email" return await _routine_creation_case( ctx, case_name="qa_2e_calendar_prep_email_routine", routine_name=routine_name, - marker=marker, + marker=None, required_text=["routine", "email"], - prompt=( - f"QA case 2E: create a routine named {routine_name}. Every weekday " - "morning, inspect my connected Google Calendar for upcoming meetings, " - "use connected Google Drive or Docs for relevant context when available, " - "include current NEAR AI news if useful, and send the meeting-prep " - "summary by Gmail email. Create the routine now; do not run it yet. " - "Do not call Google, Gmail, Calendar, Drive, Docs, or auth tools now; " - "only create the scheduled routine from these instructions. " - f"In the final answer include the exact marker {marker} and include " - "the words routine and email." - ), + prompt=_qa_sheet_prompt("qa_2e_calendar_prep_email_routine"), ) @@ -2041,7 +2138,6 @@ async def case_qa_2f_calendar_prep_email_delivery(ctx: LiveQaContext) -> ProbeRe }, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "can't send", "cannot send", @@ -2134,13 +2230,12 @@ async def case_qa_5b_drive_connect(ctx: LiveQaContext) -> ProbeResult: async def case_qa_5c_strategy_doc_knowledge_base(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_5C_STRATEGY_DOC_DONE" strategy_phrase = "Reborn QA Strategy North Star: verify live WebUIv2 tool grounding." return await _live_chat_with_extensions_case( ctx, case_name="qa_5c_strategy_doc_knowledge_base", - marker=marker, - required_text=["strategy", "WebUIv2", "grounding"], + marker=None, + required_text=["strategy"], extensions=[ { "package_id": "google-docs", @@ -2151,19 +2246,11 @@ async def case_qa_5c_strategy_doc_knowledge_base(ctx: LiveQaContext) -> ProbeRes ], }, ], - prompt=( - "QA case 5C: create a new Google Docs document titled " - f"`{marker}` with this exact strategy sentence in the body: " - f"{strategy_phrase} Then read the document content back through " - "Google Docs and answer what the strategy north star is. In the " - f"final answer include the exact marker {marker}, the word strategy, " - "the word WebUIv2, and the word grounding." - ), + prompt=_qa_sheet_prompt("qa_5c_strategy_doc_knowledge_base"), timeout=360.0, extra_details={"strategy_phrase": strategy_phrase}, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "local file", "/workspace/", @@ -2215,7 +2302,6 @@ async def case_qa_5d_slack_strategy_doc_answer(ctx: LiveQaContext) -> ProbeResul }, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "can't create", "cannot create", @@ -2264,13 +2350,7 @@ async def case_qa_5d_slack_strategy_doc_answer(ctx: LiveQaContext) -> ProbeResul ctx, channel_id=channel_id, user_id=slack_user_id, - text=( - "QA case 5D: use Google Docs to read this exact document, not a " - f"search result: {doc_url}. The document ID is `{doc_id}` and " - f"the title is `{doc_marker}`. Answer with the strategy north " - "star from that document. Do not answer from memory. " - f"Include the exact marker {slack_marker} in your Slack reply." - ), + text=f"{_qa_sheet_prompt('qa_5d_slack_strategy_doc_answer')}\nGoogle doc link: {doc_url}", event_id=f"EvREBORNQA5D{suffix}", ) observed["signed_event"] = post_result @@ -2298,9 +2378,9 @@ async def case_qa_5d_slack_strategy_doc_answer(ctx: LiveQaContext) -> ProbeResul history = await _slack_history_contains_marker( ctx, channel_id=channel_id, - marker=slack_marker, + marker=nonce, oldest_epoch=wall_started, - required_text=[nonce, "Google Docs", "grounding"], + required_text=[nonce, "strategy"], ) last_history = history if history.get("found"): @@ -2339,18 +2419,22 @@ async def case_qa_6b_sheets_connect(ctx: LiveQaContext) -> ProbeResult: async def case_qa_6c_gmail_to_sheet_live_chat(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_6C_GMAIL_TO_SHEET_DONE" return await _live_chat_with_extensions_case( ctx, case_name="qa_6c_gmail_to_sheet_live_chat", - marker=marker, - required_text=["Gmail", "Google Sheet"], + marker=None, + required_text=["ABC", "spreadsheet"], extensions=[ { "package_id": "gmail", "display_name": "Gmail", "required_tools": ["gmail.list_messages"], }, + { + "package_id": "google-drive", + "display_name": "Google Drive", + "required_tools": ["google-drive.list_files"], + }, { "package_id": "google-sheets", "display_name": "Google Sheets", @@ -2360,38 +2444,20 @@ async def case_qa_6c_gmail_to_sheet_live_chat(ctx: LiveQaContext) -> ProbeResult ], }, ], - prompt=( - "QA case 6C: use Gmail to inspect at most one recent inbox message, " - "then create a new Google Sheet named " - f"`{marker}` and write one row with columns Source, Summary, and " - "QA Marker. Use the Gmail result if one is available; if no message " - "is available, write Source as Gmail and Summary as no recent message " - "available. In the final answer include the exact marker " - f"{marker}, include the word Gmail, and include the phrase Google Sheet." - ), + prompt=_qa_sheet_prompt("qa_6c_gmail_to_sheet_live_chat"), timeout=360.0, ) async def case_qa_6d_gmail_to_sheet_routine(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_6D_GMAIL_TO_SHEET_ROUTINE_DONE" routine_name = "reborn-qa-6d-gmail-to-sheet" return await _routine_creation_case( ctx, case_name="qa_6d_gmail_to_sheet_routine", routine_name=routine_name, - marker=marker, + marker=None, required_text=["routine", "Gmail"], - prompt=( - f"QA case 6D dry-run routine-definition test: create a routine named {routine_name}. " - "When the routine runs later, it should check Gmail for new CRM or lead emails, " - "extract the sender, company or account name if present, summary, and received time, " - "then append one row to a Google Sheet CRM tracker. Create only the scheduled " - "routine definition now; do not run it, inspect accounts, verify connections, " - "or call Gmail, Google Sheets, Google auth, connector auth, or get_authenticated_user " - "tools now. In the final answer include the " - f"exact marker {marker} and include the words routine and Gmail." - ), + prompt=_qa_sheet_prompt("qa_6d_gmail_to_sheet_routine"), ) @@ -2432,7 +2498,6 @@ async def case_qa_6e_gmail_to_sheet_delivery(ctx: LiveQaContext) -> ProbeResult: timeout=420.0, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "can't create", "cannot create", @@ -2509,50 +2574,64 @@ async def _routine_creation_case( *, case_name: str, prompt: str, - marker: str, + marker: str | None, routine_name: str, required_text: list[str], + extensions: list[dict[str, object]] | None = None, + extra_details: dict[str, object] | None = None, ) -> ProbeResult: - before_count = _trigger_record_count(ctx.reborn_home, routine_name) - result = await _live_chat_case( - ctx, - case_name=case_name, - prompt=prompt, - marker=marker, - required_text=required_text, - timeout=180.0, - extra_details={ - "routine_name": routine_name, - "trigger_records_before": before_count, - }, - ) - after_count = _trigger_record_count(ctx.reborn_home, routine_name) + count_name = routine_name if marker else None + before_count = _trigger_record_count(ctx.reborn_home, count_name) + details = { + "routine_name": routine_name, + "trigger_records_before": before_count, + **(extra_details or {}), + } + if extensions: + result = await _live_chat_with_extensions_case( + ctx, + case_name=case_name, + prompt=prompt, + marker=marker, + required_text=required_text, + extensions=extensions, + timeout=180.0, + extra_details=details, + ) + else: + result = await _live_chat_case( + ctx, + case_name=case_name, + prompt=prompt, + marker=marker, + required_text=required_text, + timeout=180.0, + extra_details=details, + ) + after_count = _trigger_record_count(ctx.reborn_home, count_name) result.details["trigger_records_after"] = after_count if result.success and after_count <= before_count: result.success = False result.details["error"] = ( - f"assistant returned success marker but routine {routine_name!r} " - "was not added to trigger_records" + f"assistant returned success but routine scope {routine_name!r} " + "did not add a trigger_record" ) return result async def case_qa_3c_endpoint_status_slack_routine(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_3C_ENDPOINT_STATUS_ROUTINE_DONE" routine_name = "reborn-qa-3c-endpoint-status-slack" + prompt = _qa_sheet_prompt("qa_3c_endpoint_status_slack_routine").replace( + "[endpoint URL]", + ENDPOINT_STATUS_URL, + ) return await _routine_creation_case( ctx, case_name="qa_3c_endpoint_status_slack_routine", routine_name=routine_name, - marker=marker, + marker=None, required_text=["routine"], - prompt=( - f"QA case 3C: create a routine named {routine_name}. Every 5 minutes, " - "ping https://cloud-api.near.ai, check whether it returns HTTP 200, " - "and send the result in a Slack DM. Create the routine now; do not run " - "the check immediately. In the final answer include the exact marker " - f"{marker} and include the text routine." - ), + prompt=prompt, ) @@ -2647,7 +2726,7 @@ async def case_qa_3d_endpoint_status_slack_delivery(ctx: LiveQaContext) -> Probe routine_prefix="reborn-qa-3d-endpoint-status-slack-delivery", marker_prefix="REBORN_QA_3D_ENDPOINT_STATUS", routine_instruction=( - "check https://cloud-api.near.ai with live HTTP or web access, report " + f"check {ENDPOINT_STATUS_URL} with live HTTP or web access, report " "the observed HTTP status, and send the result to Slack" ), required_delivery_text=["status"], @@ -2655,45 +2734,25 @@ async def case_qa_3d_endpoint_status_slack_delivery(ctx: LiveQaContext) -> Probe async def case_qa_4c_github_release_live_chat(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_4C_GITHUB_RELEASE_DONE" - release = await _live_github_latest_release("nearai", "ironclaw") - api_url = release["api_url"] return await _live_chat_case( ctx, case_name="qa_4c_github_release_live_chat", - prompt=( - "QA case 4C: perform exactly one public HTTP GET to " - f"{api_url}. Do not use an authenticated GitHub connector, GitHub auth " - "flow, save/download tools, or any other URL. Confirm that the live " - f"response tag_name is {release['tag_name']}, then immediately final-answer " - f"with the exact marker {marker}, the text GitHub, and the release tag " - f"{release['tag_name']}." - ), - marker=marker, - required_text=["GitHub", release["tag_name"]], + prompt=_qa_sheet_prompt("qa_4c_github_release_live_chat"), + marker=None, + required_text=["release"], timeout=240.0, - extra_details=release, ) async def case_qa_4d_github_release_slack_routine(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_4D_GITHUB_RELEASE_SLACK_ROUTINE_DONE" routine_name = "reborn-qa-4d-github-release-slack" return await _routine_creation_case( ctx, case_name="qa_4d_github_release_slack_routine", routine_name=routine_name, - marker=marker, + marker=None, required_text=["routine"], - prompt=( - f"QA case 4D: create a routine named {routine_name}. Every 5 minutes, " - "check https://github.com/nearai/ironclaw for the latest releases and " - "send a Slack message summarizing any new release. Create the routine " - "now; do not run the check immediately. Do not call GitHub tools, " - "GitHub auth, or connector auth tools now; only create the scheduled " - "routine from these instructions. In the final answer include " - f"the exact marker {marker} and include the text routine." - ), + prompt=_qa_sheet_prompt("qa_4d_github_release_slack_routine"), ) @@ -2791,7 +2850,6 @@ async def case_qa_4e_github_release_email_delivery(ctx: LiveQaContext) -> ProbeR }, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "can't send", "cannot send", @@ -2916,7 +2974,7 @@ async def case_qa_7d_slack_bug_message_trigger(ctx: LiveQaContext) -> ProbeResul wall_started = time.time() case_name = "qa_7d_slack_bug_message_trigger" suffix = str(int(wall_started * 1000)) - marker = f"REBORN_QA_7D_SLACK_BUG_TRIGGER_{suffix}" + marker = "bug" observed: dict[str, object] = {"marker": marker} try: slack = _slack_preflight(ctx) @@ -2930,40 +2988,32 @@ async def case_qa_7d_slack_bug_message_trigger(ctx: LiveQaContext) -> ProbeResul channel_id = _slack_delivery_channel_id(ctx) if not channel_id: raise AssertionError("Slack inbound test could not resolve a DM/channel id") + if not _slack_delivery_target_is_dm(channel_id): + raise AssertionError( + "Slack bug-message trigger test must inject into a DM target; " + f"got channel_id={channel_id!r}" + ) slack_user_id = str(slack.get("legacy_actor_user_id") or "U0REBORNQA") - text = ( - f"bug: live QA signed Slack inbound test {marker}. " - "This is a plain direct-message reply test; do not call tools, do not " - "configure channels, and do not change delivery settings. " - f"Answer directly with the exact marker {marker} and the word bug." - ) + qa_sheet_prompt = _qa_sheet_prompt(case_name) + text = f"bug: reborn QA bug logger smoke {suffix}" + observed["qa_sheet_prompt"] = qa_sheet_prompt + observed["slack_event_text"] = text + event_id = f"EvREBORNQA7D{suffix}" post_result = await _post_signed_slack_dm_event( ctx, channel_id=channel_id, user_id=slack_user_id, text=text, - event_id=f"EvREBORNQA7D{suffix}", + event_id=event_id, ) observed["signed_event"] = post_result - deadline = time.monotonic() + 180.0 - last_history: dict[str, object] | None = None - while time.monotonic() < deadline: - history = await _slack_history_contains_marker( - ctx, - channel_id=channel_id, - marker=marker, - oldest_epoch=wall_started, - required_text=["bug"], - ) - last_history = history - if history.get("found"): - observed["slack_history"] = history - return _result(case_name, True, started, observed) - await asyncio.sleep(2.0) - raise AssertionError( - "Slack reply marker was not observed after signed bug: event. " - f"last_history={last_history!r}" + run_id = await _wait_for_slack_event_run_id( + ctx, + event_id=event_id, + timeout=180.0, ) + observed["accepted_run_id"] = run_id + return _result(case_name, True, started, observed) except Exception as exc: return _result(case_name, False, started, {"error": str(exc), **observed}) @@ -3001,7 +3051,6 @@ async def case_qa_7e_slack_bug_sheet_delivery(ctx: LiveQaContext) -> ProbeResult timeout=360.0, forbidden_text=[ "auth_denied", - "auth_required", "authentication required", "can't create", "cannot create", @@ -3092,53 +3141,147 @@ async def case_qa_7e_slack_bug_sheet_delivery(ctx: LiveQaContext) -> ProbeResult async def case_qa_7c_slack_bug_logger_routine(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_7C_SLACK_BUG_SHEET_ROUTINE_DONE" + started = time.monotonic() routine_name = "reborn-qa-7c-slack-bug-sheet" - return await _routine_creation_case( - ctx, - case_name="qa_7c_slack_bug_logger_routine", - routine_name=routine_name, - marker=marker, - required_text=["routine", "bug"], - prompt=( - f"QA case 7C: create a routine named {routine_name}. When a Slack " - "message in my product channel starts with `bug:`, extract the bug " - "summary, reporter, Slack timestamp, and current status, then append " - "one row to my connected Google Sheet for product bug tracking. Create " - "the routine now; do not trigger or run it yet. Do not call Slack, " - "Google Sheets, Google auth, or connector auth tools now; only create " - "the scheduled routine from these instructions. In the final answer " - f"include the exact marker {marker} and include the words routine and bug." - ), - ) + try: + access_token, token_meta = _google_runtime_access_token( + ctx.reborn_home, + _auth_user_id(), + ctx.env, + ) + sheet_fixture = await _create_google_spreadsheet_fixture( + access_token=access_token, + title=QA_7C_BUG_LOGGING_SHEET_TITLE, + values=[ + ["Summary", "Reporter", "Slack Timestamp", "Status", "QA Marker"], + ], + ) + return await _routine_creation_case( + ctx, + case_name="qa_7c_slack_bug_logger_routine", + routine_name=routine_name, + marker=None, + required_text=["trigger", "bug"], + prompt=_qa_sheet_prompt("qa_7c_slack_bug_logger_routine"), + extensions=[ + { + "package_id": "slack", + "display_name": "Slack", + "required_tools": [], + }, + { + "package_id": "google-drive", + "display_name": "Google Drive", + "required_tools": ["google-drive.list_files"], + }, + { + "package_id": "google-sheets", + "display_name": "Google Sheets", + "required_tools": [ + "google-sheets.read_values", + "google-sheets.append_values", + ], + }, + ], + extra_details={ + "google_token": token_meta, + "bug_log_sheet_fixture": sheet_fixture, + }, + ) + except Exception as exc: + return _result( + "qa_7c_slack_bug_logger_routine", + False, + started, + {"routine_name": routine_name, "error": str(exc)}, + ) async def case_qa_7a_slack_product_channel_connect(ctx: LiveQaContext) -> ProbeResult: + from playwright.async_api import expect + started = time.monotonic() - observed: dict[str, object] = {} + case_name = "qa_7a_slack_product_channel_connect" + prompt = _qa_sheet_prompt(case_name) + observed: dict[str, object] = {"chat_connect_prompt": prompt} try: slack = _slack_preflight(ctx) + delivery_channel_id = _slack_delivery_channel_id(ctx) + route_discovery = slack.get("route_discovery") + route_discovery_details = route_discovery if isinstance(route_discovery, dict) else {} observed.update( { "delivery_target_present": slack.get("delivery_target_present"), "route_configured_from_env": slack.get("route_configured_from_env"), + "slack_dm_user_source": route_discovery_details.get("dm_user_source"), + "slack_dm_user_id_present": bool(route_discovery_details.get("dm_user_id")), + "slack_delivery_channel_id_present": bool(delivery_channel_id), + "slack_delivery_target_kind": ( + "dm" if _slack_delivery_target_is_dm(delivery_channel_id) else "non_dm" + ), } ) if not slack.get("delivery_target_present"): raise AssertionError( - "Slack product-channel route is not configured for this WebUI user" + "Slack DM delivery target is not configured for this WebUI user" ) - connect_result = await _slack_connect_case( - ctx, - case_name="qa_7a_slack_product_channel_connect", - ) - observed.update(connect_result.details) - if not connect_result.success: - raise AssertionError(str(connect_result.details.get("error") or connect_result.details)) - return _result("qa_7a_slack_product_channel_connect", True, started, observed) + if not _slack_delivery_target_is_dm(delivery_channel_id): + raise AssertionError( + "Slack live QA delivery target must be a DM to the user; " + f"got channel_id={delivery_channel_id!r}" + ) + + async def action(page: object) -> None: + capability_ids = QA_7A_CHAT_CONNECT_CAPABILITY_IDS + baseline_statuses = _capability_run_statuses( + ctx.reborn_home, + capability_ids, + ) + baseline_completed = _completed_capability_counts(baseline_statuses) + observed["baseline_capability_statuses"] = baseline_statuses + await page.goto( + f"{ctx.base_url}/v2/?token={AUTH_TOKEN}", + wait_until="domcontentloaded", + ) # type: ignore[attr-defined] + composer = page.locator("[data-testid='chat-composer']") # type: ignore[attr-defined] + await expect(composer).to_be_visible(timeout=15000) + await composer.fill(prompt) + await composer.press("Enter") + await expect(page.locator("[data-testid='msg-user']").last).to_contain_text( # type: ignore[attr-defined] + prompt[:80], + timeout=15000, + ) + observed["text_excerpt"] = await _wait_for_assistant_reply( + page, + marker=None, + required_text=["slack"], + timeout=180.0, + ) + deadline = time.monotonic() + 180.0 + while time.monotonic() < deadline: + await _approve_visible_tool_gate(page) + statuses = _capability_run_statuses(ctx.reborn_home, capability_ids) + observed["capability_statuses"] = statuses + completed = _completed_capability_counts(statuses) + missing = [ + capability_id + for capability_id in capability_ids + if completed.get(capability_id, 0) + <= baseline_completed.get(capability_id, 0) + ] + if not missing: + return + await asyncio.sleep(1.0) + raise AssertionError( + "Slack DM connect prompt did not complete expected capabilities: " + f"{capability_ids!r}; observed statuses={observed.get('capability_statuses')!r}" + ) + + await _with_page(ctx.output_dir, case_name, action) + return _result(case_name, True, started, observed) except Exception as exc: return _result( - "qa_7a_slack_product_channel_connect", + case_name, False, started, {"error": str(exc), **observed}, @@ -3146,20 +3289,12 @@ async def case_qa_7a_slack_product_channel_connect(ctx: LiveQaContext) -> ProbeR async def case_qa_8b_hn_keyword_live_chat(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN8BHNSEARCHDONE" return await _live_chat_case( ctx, case_name="qa_8b_hn_keyword_live_chat", - prompt=( - "Task 8B: perform exactly one public HTTP GET to the Hacker News Algolia " - f"API URL {HN_KEYWORD_SEARCH_URL}. Treat that response as the live Hacker " - "News keyword probe for recent NEAR AI posts. Do not use web_search, " - "authenticated connectors, save/download tools, or any other URL. Then " - f"immediately final-answer with the exact marker {marker} and include the " - "text Hacker News." - ), - marker=marker, - required_text=["Hacker News"], + prompt=_qa_sheet_prompt("qa_8b_hn_keyword_live_chat"), + marker=None, + required_text=["news.ycombinator.com"], timeout=240.0, ) @@ -3169,23 +3304,14 @@ async def case_qa_8a_slack_connect(ctx: LiveQaContext) -> ProbeResult: async def case_qa_8c_hn_keyword_slack_routine(ctx: LiveQaContext) -> ProbeResult: - marker = "REBORN_QA_8C_HN_SLACK_ROUTINE_DONE" routine_name = "reborn-qa-8c-hn-keyword-slack" return await _routine_creation_case( ctx, case_name="qa_8c_hn_keyword_slack_routine", routine_name=routine_name, - marker=marker, + marker=None, required_text=["routine"], - prompt=( - f"QA case 8C: create a routine named {routine_name}. Every hour, " - "check Hacker News for new posts mentioning IronClaw or NEAR AI and " - "send a summary to Slack. Create the routine now; do not run the " - "search immediately. Do not call Slack delivery or auth tools now; " - "only create the scheduled routine from these instructions. In the " - "final answer include the exact marker " - f"{marker} and include the text routine." - ), + prompt=_qa_sheet_prompt("qa_8c_hn_keyword_slack_routine"), ) diff --git a/scripts/reborn_webui_v2_live_qa/slack_helpers.py b/scripts/reborn_webui_v2_live_qa/slack_helpers.py index eab6b2ec8bb..df3f3d97be0 100644 --- a/scripts/reborn_webui_v2_live_qa/slack_helpers.py +++ b/scripts/reborn_webui_v2_live_qa/slack_helpers.py @@ -192,13 +192,24 @@ def _discover_slack_dm_route_channel( token = _env_value(bot_env, extra_env) if not token: return {"checked": False, "ok": False, "error": "bot token env unavailable"} + configured_route_user_id = os.environ.get( + "REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_USER_ID", + "", + ).strip() + inbound_user_id = os.environ.get( + "REBORN_WEBUI_V2_LIVE_QA_SLACK_INBOUND_USER_ID", + "", + ).strip() + if inbound_user_id == "U0REBORNQA": + inbound_user_id = "" + dm_user_id = configured_route_user_id or inbound_user_id or "USLACKBOT" try: import httpx response = httpx.post( "https://slack.com/api/conversations.open", headers={"Authorization": f"Bearer {token}"}, - data={"users": "USLACKBOT"}, + data={"users": dm_user_id}, timeout=20.0, ) payload = response.json() @@ -212,6 +223,8 @@ def _discover_slack_dm_route_channel( result: dict[str, object] = { "checked": True, "ok": bool(payload.get("ok")), + "dm_user_id": dm_user_id, + "dm_user_source": "env" if dm_user_id != "USLACKBOT" else "fallback_slackbot", } channel = payload.get("channel") if isinstance(channel, dict): diff --git a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py index 93789736703..06dad2830e3 100644 --- a/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py +++ b/scripts/reborn_webui_v2_live_qa/test_run_live_qa.py @@ -17,6 +17,7 @@ import sqlite3 import sys import tempfile +import types import unittest from pathlib import Path from unittest.mock import patch @@ -191,10 +192,11 @@ def fake_capability_run_statuses(_reborn_home, capability_ids): self.assertEqual(set(captured_registry), set(cases)) for case_name, (_case_fn, package_id, verification_caps) in cases.items(): prompt = str(captured_chat[case_name]["prompt"]) - self.assertIn("from this chat", prompt) - self.assertIn("extension_search", prompt) - self.assertIn(f"`{package_id}`", prompt) + self.assertEqual(prompt, run_live_qa.QA_SHEET_PROMPTS[case_name]) + self.assertNotIn("extension_search", prompt) + self.assertNotIn(f"`{package_id}`", prompt) self.assertNotIn("/v2/extensions/registry", prompt) + self.assertIsNone(captured_chat[case_name]["marker"]) extra_details = captured_chat[case_name]["extra_details"] self.assertIsInstance(extra_details, dict) self.assertTrue(extra_details["chat_connect_flow"]) @@ -203,7 +205,12 @@ def fake_capability_run_statuses(_reborn_home, capability_ids): self.assertIn(run_live_qa.EXTENSION_INSTALL_CAPABILITY_ID, required_capabilities) self.assertIn(run_live_qa.EXTENSION_ACTIVATE_CAPABILITY_ID, required_capabilities) for capability_id in verification_caps: - self.assertIn(capability_id, required_capabilities) + self.assertNotIn(capability_id, required_capabilities) + self.assertEqual( + extra_details["verification_capabilities"], + verification_caps, + ) + self.assertFalse(extra_details["verification_capabilities_required"]) self.assertFalse(captured_registry[case_name]["ensure_installed"]) def test_product_connect_case_fails_when_chat_does_not_use_extension_lifecycle(self): @@ -252,6 +259,539 @@ def fake_capability_run_statuses(_reborn_home, capability_ids): str(result.details["error"]), ) + def test_routine_creation_case_fails_when_no_trigger_is_created(self): + captured_prompts: list[str] = [] + + async def fake_live_chat_case(_ctx, **kwargs): + captured_prompts.append(kwargs["prompt"]) + extra_details = kwargs.get("extra_details") or {} + return run_live_qa.ProbeResult( + provider="test", + mode=f"live:{kwargs['case_name']}", + success=True, + latency_ms=1, + details={ + "text_excerpt": "routine created", + **extra_details, + }, + ) + + with ( + patch.object( + run_live_qa, + "_live_chat_case", + side_effect=fake_live_chat_case, + ), + patch.object(run_live_qa, "_trigger_record_count", side_effect=[0, 0]), + ): + result = asyncio.run( + run_live_qa._routine_creation_case( + self._dummy_ctx(), + case_name="qa_test_routine", + prompt="original sheet prompt", + marker=None, + routine_name="qa-test-routine", + required_text=["routine"], + ) + ) + + self.assertFalse(result.success) + self.assertEqual(captured_prompts, ["original sheet prompt"]) + self.assertEqual(result.details["trigger_records_after"], 0) + self.assertIn("did not add a trigger_record", result.details["error"]) + + def test_routine_creation_case_can_preinstall_extensions(self): + captured: dict[str, object] = {} + + async def fake_live_chat_with_extensions_case( + _ctx, + *, + case_name, + prompt, + marker, + required_text, + extensions, + timeout, + extra_details, + ): + captured.update( + { + "case_name": case_name, + "prompt": prompt, + "marker": marker, + "required_text": required_text, + "extensions": extensions, + "timeout": timeout, + "extra_details": extra_details, + } + ) + extra_details = extra_details or {} + return run_live_qa.ProbeResult( + provider="test", + mode=f"live:{case_name}", + success=True, + latency_ms=1, + details={ + "text_excerpt": "routine created", + **extra_details, + }, + ) + + with ( + patch.object( + run_live_qa, + "_live_chat_with_extensions_case", + side_effect=fake_live_chat_with_extensions_case, + ), + patch.object(run_live_qa, "_trigger_record_count", side_effect=[0, 1]), + ): + result = asyncio.run( + run_live_qa._routine_creation_case( + self._dummy_ctx(), + case_name="qa_test_routine", + prompt="original sheet prompt", + marker=None, + routine_name="qa-test-routine", + required_text=["routine"], + extensions=[ + { + "package_id": "google-sheets", + "display_name": "Google Sheets", + "required_tools": ["google-sheets.append_values"], + } + ], + extra_details={"fixture_ready": True}, + ) + ) + + self.assertTrue(result.success) + self.assertEqual(captured["case_name"], "qa_test_routine") + self.assertEqual(captured["prompt"], "original sheet prompt") + self.assertIsNone(captured["marker"]) + self.assertEqual(captured["required_text"], ["routine"]) + self.assertEqual(captured["extensions"][0]["package_id"], "google-sheets") + self.assertEqual(captured["timeout"], 180.0) + self.assertTrue(result.details["fixture_ready"]) + + def test_slack_delivery_target_dm_detection(self): + self.assertTrue(run_live_qa._slack_delivery_target_is_dm("D12345")) + self.assertFalse(run_live_qa._slack_delivery_target_is_dm("C12345")) + self.assertFalse(run_live_qa._slack_delivery_target_is_dm(None)) + + def test_slack_dm_route_discovery_prefers_configured_user(self): + captured: dict[str, object] = {} + + class FakeResponse: + def json(self): + return { + "ok": True, + "channel": { + "id": "DQAUSER", + "is_im": True, + }, + } + + def fake_post(_url, **kwargs): + captured.update(kwargs) + return FakeResponse() + + fake_httpx = types.SimpleNamespace(post=fake_post) + with ( + patch.dict( + os.environ, + {"REBORN_WEBUI_V2_LIVE_QA_SLACK_ROUTE_USER_ID": "UQAUSER"}, + clear=True, + ), + patch.dict(sys.modules, {"httpx": fake_httpx}), + ): + result = run_live_qa._discover_slack_dm_route_channel( + "[slack]\nbot_token_env = \"SLACK_BOT_TOKEN\"\n", + {"SLACK_BOT_TOKEN": "xoxb-test"}, + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["dm_user_id"], "UQAUSER") + self.assertEqual(result["dm_user_source"], "env") + self.assertEqual(captured["data"]["users"], "UQAUSER") + + def test_slack_dm_route_discovery_ignores_synthetic_inbound_user(self): + captured: dict[str, object] = {} + + class FakeResponse: + def json(self): + return { + "ok": True, + "channel": { + "id": "DSLACKBOT", + "is_im": True, + }, + } + + def fake_post(_url, **kwargs): + captured.update(kwargs) + return FakeResponse() + + fake_httpx = types.SimpleNamespace(post=fake_post) + with ( + patch.dict( + os.environ, + {"REBORN_WEBUI_V2_LIVE_QA_SLACK_INBOUND_USER_ID": "U0REBORNQA"}, + clear=True, + ), + patch.dict(sys.modules, {"httpx": fake_httpx}), + ): + result = run_live_qa._discover_slack_dm_route_channel( + "[slack]\nbot_token_env = \"SLACK_BOT_TOKEN\"\n", + {"SLACK_BOT_TOKEN": "xoxb-test"}, + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["dm_user_id"], "USLACKBOT") + self.assertEqual(result["dm_user_source"], "fallback_slackbot") + self.assertEqual(captured["data"]["users"], "USLACKBOT") + + def test_qa_7a_requires_dm_delivery_target(self): + with ( + patch.object( + run_live_qa, + "_slack_preflight", + return_value={ + "delivery_target_present": True, + "route_configured_from_env": True, + }, + ), + patch.object(run_live_qa, "_slack_delivery_channel_id", return_value="C12345"), + ): + result = asyncio.run( + run_live_qa.case_qa_7a_slack_product_channel_connect(self._dummy_ctx()) + ) + + self.assertFalse(result.success) + self.assertIn("must be a DM", str(result.details["error"])) + self.assertEqual(result.details["slack_delivery_target_kind"], "non_dm") + + def test_qa_7a_connect_capabilities_match_chat_connect_flow(self): + self.assertEqual( + run_live_qa.QA_7A_CHAT_CONNECT_CAPABILITY_IDS, + [ + run_live_qa.EXTENSION_SEARCH_CAPABILITY_ID, + run_live_qa.EXTENSION_INSTALL_CAPABILITY_ID, + run_live_qa.EXTENSION_ACTIVATE_CAPABILITY_ID, + ], + ) + self.assertNotIn( + run_live_qa.OUTBOUND_DELIVERY_TARGETS_LIST_CAPABILITY_ID, + run_live_qa.QA_7A_CHAT_CONNECT_CAPABILITY_IDS, + ) + + def test_qa_7a_requires_new_connect_capability_completions(self): + class FakeLocator: + @property + def last(self): + return self + + async def fill(self, _text): + return None + + async def press(self, _key): + return None + + class FakePage: + async def goto(self, _url, **_kwargs): + return None + + def locator(self, _selector): + return FakeLocator() + + class FakeExpectation: + async def to_be_visible(self, **_kwargs): + return None + + async def to_contain_text(self, _text, **_kwargs): + return None + + capability_ids = run_live_qa.QA_7A_CHAT_CONNECT_CAPABILITY_IDS + baseline = {capability_id: ["completed"] for capability_id in capability_ids} + stale = {capability_id: ["completed"] for capability_id in capability_ids} + fresh = { + capability_id: ["completed", "completed"] + for capability_id in capability_ids + } + status_sequence = [baseline, stale, fresh] + + async def fake_with_page(_output_dir, _case_name, action): + await action(FakePage()) + + async def fake_wait_for_assistant_reply(_page, **_kwargs): + return "Slack is connected" + + async def fake_approve_visible_tool_gate(_page): + return None + + async def fake_sleep(_seconds): + return None + + def fake_capability_run_statuses(_reborn_home, _capability_ids): + return status_sequence.pop(0) if status_sequence else fresh + + with ( + patch.object( + run_live_qa, + "_slack_preflight", + return_value={ + "delivery_target_present": True, + "route_configured_from_env": True, + }, + ), + patch.object(run_live_qa, "_slack_delivery_channel_id", return_value="D12345"), + patch.object(run_live_qa, "_with_page", side_effect=fake_with_page), + patch.object( + run_live_qa, + "_wait_for_assistant_reply", + side_effect=fake_wait_for_assistant_reply, + ), + patch.object( + run_live_qa, + "_approve_visible_tool_gate", + side_effect=fake_approve_visible_tool_gate, + ), + patch.object( + run_live_qa, + "_capability_run_statuses", + side_effect=fake_capability_run_statuses, + ), + patch.object(run_live_qa.asyncio, "sleep", side_effect=fake_sleep), + patch("playwright.async_api.expect", return_value=FakeExpectation()), + ): + result = asyncio.run( + run_live_qa.case_qa_7a_slack_product_channel_connect(self._dummy_ctx()) + ) + + self.assertTrue(result.success) + self.assertEqual(result.details["baseline_capability_statuses"], baseline) + self.assertEqual(result.details["capability_statuses"], fresh) + self.assertEqual(result.details["text_excerpt"], "Slack is connected") + + def test_completed_capability_counts_ignore_stale_completed_runs(self): + counts = run_live_qa._completed_capability_counts( + { + "extension_search": ["completed", "failed", "completed"], + "extension_install": ["running"], + "extension_activate": [], + } + ) + + self.assertEqual(counts["extension_search"], 2) + self.assertEqual(counts["extension_install"], 0) + self.assertEqual(counts["extension_activate"], 0) + + def test_qa_7c_prepares_bug_logging_sheet_before_sheet_prompt(self): + captured_fixture: dict[str, object] = {} + captured_routine: dict[str, object] = {} + + async def fake_create_google_spreadsheet_fixture( + *, + access_token, + title, + values, + sheet_name="Sheet1", + ): + captured_fixture.update( + { + "access_token": access_token, + "title": title, + "values": values, + "sheet_name": sheet_name, + } + ) + return { + "spreadsheet_id": "sheet-123", + "spreadsheet_url": "https://docs.google.com/spreadsheets/d/sheet-123/edit", + "title": title, + } + + async def fake_routine_creation_case( + _ctx, + *, + case_name, + prompt, + marker, + routine_name, + required_text, + extensions, + extra_details, + ): + captured_routine.update( + { + "case_name": case_name, + "prompt": prompt, + "marker": marker, + "routine_name": routine_name, + "required_text": required_text, + "extensions": extensions, + "extra_details": extra_details, + } + ) + extra_details = extra_details or {} + return run_live_qa.ProbeResult( + provider="test", + mode="live:qa_7c_slack_bug_logger_routine", + success=True, + latency_ms=1, + details={"text_excerpt": "routine bug created", **extra_details}, + ) + + with ( + patch.object( + run_live_qa, + "_google_runtime_access_token", + return_value=("fresh-access-token", {"source": "test"}), + ), + patch.object( + run_live_qa, + "_create_google_spreadsheet_fixture", + side_effect=fake_create_google_spreadsheet_fixture, + ), + patch.object( + run_live_qa, + "_routine_creation_case", + side_effect=fake_routine_creation_case, + ), + ): + result = asyncio.run( + run_live_qa.case_qa_7c_slack_bug_logger_routine(self._dummy_ctx()) + ) + + self.assertTrue(result.success) + self.assertEqual(captured_fixture["access_token"], "fresh-access-token") + self.assertEqual( + captured_fixture["title"], + run_live_qa.QA_7C_BUG_LOGGING_SHEET_TITLE, + ) + self.assertEqual(captured_fixture["sheet_name"], "Sheet1") + self.assertEqual( + captured_fixture["values"], + [["Summary", "Reporter", "Slack Timestamp", "Status", "QA Marker"]], + ) + self.assertEqual(captured_routine["case_name"], "qa_7c_slack_bug_logger_routine") + self.assertIsNone(captured_routine["marker"]) + self.assertEqual(captured_routine["routine_name"], "reborn-qa-7c-slack-bug-sheet") + self.assertEqual(captured_routine["required_text"], ["trigger", "bug"]) + self.assertEqual( + captured_routine["prompt"], + run_live_qa._qa_sheet_prompt("qa_7c_slack_bug_logger_routine"), + ) + package_ids = [ + extension["package_id"] for extension in captured_routine["extensions"] + ] + self.assertEqual(package_ids, ["slack", "google-drive", "google-sheets"]) + self.assertEqual( + captured_routine["extra_details"]["bug_log_sheet_fixture"]["spreadsheet_id"], + "sheet-123", + ) + + def test_qa_7d_accepts_signed_slack_event_into_reborn_run(self): + captured_event: dict[str, object] = {} + + async def fake_post_signed_slack_dm_event(_ctx, **kwargs): + captured_event.update(kwargs) + return { + "status_code": 200, + "event_id": kwargs["event_id"], + "channel_id_present": True, + } + + async def fake_wait_for_slack_event_run_id(_ctx, **kwargs): + return f"run-for-{kwargs['event_id']}" + + with ( + patch.object( + run_live_qa, + "_slack_preflight", + return_value={ + "legacy_actor_configured": True, + "legacy_actor_user_id": "U0REBORNQA", + "delivery_target_present": True, + }, + ), + patch.object(run_live_qa, "_slack_delivery_channel_id", return_value="D12345"), + patch.object( + run_live_qa, + "_post_signed_slack_dm_event", + side_effect=fake_post_signed_slack_dm_event, + ), + patch.object( + run_live_qa, + "_wait_for_slack_event_run_id", + side_effect=fake_wait_for_slack_event_run_id, + ), + ): + result = asyncio.run( + run_live_qa.case_qa_7d_slack_bug_message_trigger(self._dummy_ctx()) + ) + + self.assertTrue(result.success) + self.assertTrue(result.details["accepted_run_id"].startswith("run-for-")) + self.assertEqual(result.details["signed_event"]["status_code"], 200) + self.assertTrue( + str(captured_event["text"]).startswith("bug: reborn QA bug logger smoke ") + ) + self.assertNotIn("In Slack", str(captured_event["text"])) + self.assertEqual(result.details["slack_event_text"], captured_event["text"]) + self.assertEqual( + result.details["qa_sheet_prompt"], + run_live_qa._qa_sheet_prompt("qa_7d_slack_bug_message_trigger"), + ) + + def test_endpoint_status_routine_prompt_uses_real_endpoint(self): + captured: dict[str, object] = {} + + async def fake_live_chat_case(_ctx, **kwargs): + captured.update(kwargs) + return run_live_qa.ProbeResult( + provider="test", + mode=f"live:{kwargs['case_name']}", + success=True, + latency_ms=1, + details={"text_excerpt": "Routine created"}, + ) + + with ( + patch.object(run_live_qa, "_live_chat_case", side_effect=fake_live_chat_case), + patch.object(run_live_qa, "_trigger_record_count", side_effect=[0, 1]), + ): + result = asyncio.run( + run_live_qa.case_qa_3c_endpoint_status_slack_routine(self._dummy_ctx()) + ) + + self.assertTrue(result.success) + prompt = str(captured["prompt"]) + self.assertNotIn("[endpoint URL]", prompt) + self.assertIn(run_live_qa.ENDPOINT_STATUS_URL, prompt) + self.assertIsNone(captured["marker"]) + self.assertEqual(captured["required_text"], ["routine"]) + + def test_hn_live_chat_accepts_hacker_news_url_host(self): + captured: dict[str, object] = {} + + async def fake_live_chat_case(_ctx, **kwargs): + captured.update(kwargs) + return run_live_qa.ProbeResult( + provider="test", + mode=f"live:{kwargs['case_name']}", + success=True, + latency_ms=1, + details={"text_excerpt": "news.ycombinator.com/item?id=1"}, + ) + + with patch.object(run_live_qa, "_live_chat_case", side_effect=fake_live_chat_case): + result = asyncio.run( + run_live_qa.case_qa_8b_hn_keyword_live_chat(self._dummy_ctx()) + ) + + self.assertTrue(result.success) + self.assertEqual(captured["required_text"], ["news.ycombinator.com"]) + def test_live_google_side_effect_cases_install_required_extensions(self): captured: dict[str, dict[str, object]] = {} spreadsheet_id = "1AbCdEfGhIjKlMnOpQrStUvWxYz_1234567890" @@ -362,6 +902,10 @@ async def fake_google_sheet_contains_marker(**_kwargs): captured["qa_6e_gmail_to_sheet_delivery"]["required_text"], ["Google Sheet"], ) + self.assertEqual( + captured["qa_6c_gmail_to_sheet_live_chat"]["required_text"], + ["ABC", "spreadsheet"], + ) self.assertTrue( extensions_by_case["qa_2f_calendar_prep_email_delivery"]["google-docs"].get( "ensure_installed", @@ -380,6 +924,12 @@ async def fake_google_sheet_contains_marker(**_kwargs): True, ) ) + self.assertEqual( + extensions_by_case["qa_6c_gmail_to_sheet_live_chat"]["google-drive"].get( + "required_tools", + ), + ["google-drive.list_files"], + ) self.assertTrue( extensions_by_case["qa_6e_gmail_to_sheet_delivery"]["gmail"].get( "ensure_installed", @@ -449,6 +999,7 @@ async def fake_google_sheet_contains_marker(**kwargs): def test_slack_side_effect_setup_prompts_avoid_connect_action_trigger(self): captured_prompts: dict[str, str] = {} + captured_slack_required_text: list[str] = [] document_id = "1DocCdEfGhIjKlMnOpQrStUvWxYz_1234567890" spreadsheet_id = "1AbCdEfGhIjKlMnOpQrStUvWxYz_1234567890" @@ -471,7 +1022,8 @@ async def fake_live_chat_with_extensions_case(_ctx, **kwargs): async def fake_post_signed_slack_dm_event(*_args, **_kwargs): return {"ok": True} - async def fake_slack_history_contains_marker(*_args, **_kwargs): + async def fake_slack_history_contains_marker(*_args, **kwargs): + captured_slack_required_text.extend(kwargs["required_text"]) return {"found": True} async def fake_wait_for_google_sheet_marker(*_args, **_kwargs): @@ -532,6 +1084,12 @@ async def fake_wait_for_google_sheet_marker(*_args, **_kwargs): trigger.search(prompt), f"{case_name} prompt should not trigger WebUI connect action: {prompt}", ) + self.assertIn("strategy", captured_slack_required_text) + self.assertNotIn("Google Docs", captured_slack_required_text) + self.assertNotIn("grounding", captured_slack_required_text) + self.assertTrue( + any(text.startswith("QA5D-NONCE-") for text in captured_slack_required_text) + ) def test_signed_slack_event_cases_configure_legacy_actor(self): for case_name in ( @@ -1004,6 +1562,58 @@ def test_non_telegram_qa_suite_selects_full_current_live_target(self): ): self.assertIn(case_name, selected_cases) + def test_live_canary_workflow_shards_cover_non_telegram_qa_suite(self): + args = argparse.Namespace( + all_cases=False, + non_telegram_qa_cases=True, + case=[], + ) + selected_cases = run_live_qa._selected_case_names(args) + workflow_path = ( + Path(__file__).resolve().parents[2] / ".github/workflows/live-canary.yml" + ) + workflow = workflow_path.read_text(encoding="utf-8") + match = re.search( + r"(?ms)^ reborn-webui-v2-live-qa:\n(?P.*?)^ persona-rotating:", + workflow, + ) + self.assertIsNotNone(match, "Reborn WebUI v2 live QA job missing") + + shard_case_lines = re.findall(r"^\s+cases:\s*(\S+)\s*$", match.group("body"), re.M) + self.assertEqual(len(shard_case_lines), 7) + sharded_cases = [ + case_name + for line in shard_case_lines + for case_name in line.split(",") + if case_name + ] + + self.assertEqual(len(sharded_cases), len(set(sharded_cases))) + self.assertEqual(sharded_cases, selected_cases) + all_shard_cases_match = re.search( + r"(?ms)^\s+ALL_SHARD_CASES:\s*>-\n(?P.*?)(?=^\s+run:\s*\|)", + match.group("body"), + ) + self.assertIsNotNone( + all_shard_cases_match, + "Reborn WebUI v2 live QA all-case validation list missing", + ) + all_shard_cases = [ + case_name.strip() + for line in all_shard_cases_match.group("cases").splitlines() + for case_name in line.split(",") + if case_name.strip() + ] + self.assertEqual(all_shard_cases, selected_cases) + self.assertIn( + "Unknown Reborn WebUI v2 live QA case", + match.group("body"), + ) + self.assertIn( + "target_ref is disabled for reborn-webui-v2-live-qa", + match.group("body"), + ) + def test_case_manifest_distinguishes_targeted_from_placeholder_gates(self): with tempfile.TemporaryDirectory() as tmpdir: output_dir = Path(tmpdir) diff --git a/tools-src/google-drive/google-drive-tool.capabilities.json b/tools-src/google-drive/google-drive-tool.capabilities.json index 74028eb7c49..705199de0ac 100644 --- a/tools-src/google-drive/google-drive-tool.capabilities.json +++ b/tools-src/google-drive/google-drive-tool.capabilities.json @@ -1,7 +1,7 @@ { "version": "0.2.0", "wit_version": "0.3.0", - "description": "Search, access, upload, share, and organize files and folders in Google Drive. Supports personal drives and shared (organizational) drives.", + "description": "Search, access, upload, share, and organize files and folders in Google Drive, including finding Google Sheets/spreadsheets, Docs, and Slides by name or title. Supports personal drives and shared (organizational) drives.", "http": { "allowlist": [ { diff --git a/tools-src/google-drive/src/lib.rs b/tools-src/google-drive/src/lib.rs index 3aff8066e6b..4a81397641b 100644 --- a/tools-src/google-drive/src/lib.rs +++ b/tools-src/google-drive/src/lib.rs @@ -1,7 +1,8 @@ //! Google Drive WASM Tool for IronClaw. //! //! Provides Google Drive integration for searching, accessing, uploading, -//! sharing, and organizing files and folders. Supports both personal and +//! sharing, and organizing files and folders, including finding Google Sheets/ +//! spreadsheets, Docs, and Slides by name/title. Supports both personal and //! shared (organizational) drives. //! //! # Capabilities Required @@ -28,6 +29,7 @@ //! //! ```json //! {"action": "list_files", "query": "name contains 'report' and mimeType = 'application/pdf'"} +//! {"action": "list_files", "query": "name = '' and mimeType = 'application/vnd.google-apps.spreadsheet' and trashed = false"} //! {"action": "list_files", "corpora": "drive", "drive_id": "0ABcd...", "query": "trashed = false"} //! {"action": "share_file", "file_id": "abc123", "email": "alice@company.com", "role": "writer"} //! ``` @@ -70,7 +72,8 @@ impl exports::near::agent::tool::Guest for GoogleDriveTool { fn description() -> String { "Google Drive integration for searching, accessing, uploading, sharing, and organizing \ - files and folders. Supports personal drives and shared (organizational) drives via the \ + files and folders, including finding Google Sheets/spreadsheets, Docs, and Slides by \ + name or title. Supports personal drives and shared (organizational) drives via the \ corpora parameter. Can search with Drive query syntax, download text files, upload new \ files, manage folder structure, and control sharing permissions. Requires a Google OAuth \ token with the drive scope. \ diff --git a/tools-src/google-sheets/google-sheets-tool.capabilities.json b/tools-src/google-sheets/google-sheets-tool.capabilities.json index b960c0ea593..ed880c65a6d 100644 --- a/tools-src/google-sheets/google-sheets-tool.capabilities.json +++ b/tools-src/google-sheets/google-sheets-tool.capabilities.json @@ -1,7 +1,7 @@ { "version": "0.2.0", "wit_version": "0.3.0", - "description": "Create, read, write, and format Google Sheets spreadsheets. Supports cell operations using A1 notation, sheet (tab) management, and cell formatting.", + "description": "Create, read, write, and format Google Sheets spreadsheets. Supports cell operations using A1 notation, sheet (tab) management, and cell formatting. Use Google Drive to find existing spreadsheets by name or title.", "http": { "allowlist": [ { diff --git a/tools-src/google-sheets/src/lib.rs b/tools-src/google-sheets/src/lib.rs index 1450fa93098..4cfc8f03445 100644 --- a/tools-src/google-sheets/src/lib.rs +++ b/tools-src/google-sheets/src/lib.rs @@ -78,7 +78,7 @@ impl exports::near::agent::tool::Guest for GoogleSheetsTool { Supports cell value operations (read, write, append, clear) using A1 notation, sheet \ (tab) management (add, delete, rename), and cell formatting (bold, colors, alignment, \ number formats). Spreadsheet IDs are the same as Google Drive file IDs, so use the \ - google-drive tool to search for existing spreadsheets. Requires a Google OAuth token \ + google-drive tool to search for existing spreadsheets by name or title. Requires a Google OAuth token \ with the spreadsheets scope. \ To discover all available API operations, use http GET to fetch \ (public, no auth needed)."