diff --git a/tests/e2e/emulate_provider.py b/tests/e2e/emulate_provider.py index e58f14201e7..1bf902b8b90 100644 --- a/tests/e2e/emulate_provider.py +++ b/tests/e2e/emulate_provider.py @@ -30,14 +30,21 @@ def gmail_header(message: dict, name: str) -> str | None: return None -def raw_mime(*, to: str, subject: str, body: str) -> str: - message = ( - f"To: {to}\r\n" - f"Subject: {subject}\r\n" - "Content-Type: text/plain; charset=utf-8\r\n" - "\r\n" - f"{body}" - ) +def raw_mime( + *, + to: str, + subject: str, + body: str, + in_reply_to: str | None = None, + references: str | None = None, +) -> str: + headers = [f"To: {to}", f"Subject: {subject}"] + if in_reply_to: + headers.append(f"In-Reply-To: {in_reply_to}") + if references: + headers.append(f"References: {references}") + headers.append("Content-Type: text/plain; charset=utf-8") + message = "\r\n".join([*headers, "", body]) return base64.urlsafe_b64encode(message.encode("utf-8")).decode("ascii").rstrip("=") diff --git a/tests/e2e/fixtures/emulate/google_gmail.yaml b/tests/e2e/fixtures/emulate/google_gmail.yaml index 8e80d9c4d8d..b3c291ceb8f 100644 --- a/tests/e2e/fixtures/emulate/google_gmail.yaml +++ b/tests/e2e/fixtures/emulate/google_gmail.yaml @@ -41,6 +41,8 @@ google: name: Secondary Google User messages: - id: msg_emulate_unread + thread_id: thr_emulate_unread + message_id: "" user_email: e2e.google@example.com from: qa-sender@example.com to: e2e.google@example.com diff --git a/tests/e2e/fixtures/provider_capability_coverage.toml b/tests/e2e/fixtures/provider_capability_coverage.toml index ef815c13600..7f64d2396e3 100644 --- a/tests/e2e/fixtures/provider_capability_coverage.toml +++ b/tests/e2e/fixtures/provider_capability_coverage.toml @@ -10,17 +10,22 @@ tested = [ "gmail.create_draft", "gmail.get_message", "gmail.list_messages", + "gmail.reply_to_message", "gmail.send_message", "gmail.trash_message", + "google-calendar.create_event", + "google-calendar.delete_event", "google-calendar.list_calendars", "google-calendar.list_events", "google-docs.create_document", "google-docs.insert_text", "google-docs.read_content", + "google-drive.create_folder", "google-drive.download_file", "google-drive.get_file", "google-drive.list_files", "google-drive.update_file", + "google-drive.upload_file", "google-sheets.append_values", "google-sheets.create_spreadsheet", "google-sheets.get_spreadsheet", @@ -92,10 +97,7 @@ capabilities = [ "github.unresolve_review_thread", "github.update_issue", "github.update_pull_request", - "gmail.reply_to_message", "google-calendar.add_attendees", - "google-calendar.create_event", - "google-calendar.delete_event", "google-calendar.find_free_slots", "google-calendar.get_event", "google-calendar.set_reminder", @@ -108,14 +110,12 @@ capabilities = [ "google-docs.get_document", "google-docs.insert_table", "google-docs.replace_text", - "google-drive.create_folder", "google-drive.delete_file", "google-drive.list_permissions", "google-drive.list_shared_drives", "google-drive.remove_permission", "google-drive.share_file", "google-drive.trash_file", - "google-drive.upload_file", "google-sheets.add_sheet", "google-sheets.batch_read_values", "google-sheets.clear_values", diff --git a/tests/e2e/provider_operation_cases.py b/tests/e2e/provider_operation_cases.py index c89a71a7225..133ca02baee 100644 --- a/tests/e2e/provider_operation_cases.py +++ b/tests/e2e/provider_operation_cases.py @@ -1,151 +1,6 @@ -"""Typed full-path provider operation cases and provider-side oracles.""" +"""Registry for typed full-path provider operation cases.""" -from collections.abc import Awaitable, Callable -from dataclasses import dataclass -import json -from typing import Literal +from provider_operation_google_cases import GOOGLE_PROVIDER_OPERATION_CASES +from provider_operation_types import ProviderOperationCase -import httpx - -from emulate_provider import gmail_header, google_headers, raw_mime - -BaselineAssertion = Callable[[str], Awaitable[None]] -OutcomeAssertion = Callable[[str, dict], Awaitable[None]] -ProviderService = Literal["google", "github", "slack"] - - -@dataclass(frozen=True) -class ProviderOperationCase: - """One capability invocation with provider-observable proof.""" - - case_id: str - provider_service: ProviderService - capability_id: str - arguments: dict - assert_baseline: BaselineAssertion - assert_outcome: OutcomeAssertion - - -async def _drive_file(emulate_url: str, file_id: str) -> dict: - async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client: - response = await client.get(f"{emulate_url}/drive/v3/files/{file_id}") - response.raise_for_status() - return response.json() - - -async def _assert_drive_get_baseline(emulate_url: str) -> None: - file = await _drive_file(emulate_url, "drv_reborn_qa_brief") - assert file["name"] == "Reborn QA Brief", file - - -async def _assert_drive_get_outcome(emulate_url: str, preview: dict) -> None: - await _assert_drive_get_baseline(emulate_url) - assert "Reborn QA Brief" in json.dumps(preview), preview - - -async def _assert_drive_update_baseline(emulate_url: str) -> None: - await _assert_drive_get_baseline(emulate_url) - - -async def _assert_drive_update_outcome(emulate_url: str, preview: dict) -> None: - file = await _drive_file(emulate_url, "drv_reborn_qa_brief") - assert file["name"] == "REBORN_PROVIDER_CASE_UPDATED_FILE", file - assert "REBORN_PROVIDER_CASE_UPDATED_FILE" in json.dumps(preview), preview - - -async def _drafts(emulate_url: str) -> list[dict]: - async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client: - response = await client.get( - f"{emulate_url}/gmail/v1/users/me/drafts", - params={"maxResults": 100}, - ) - response.raise_for_status() - return response.json().get("drafts", []) - - -async def _assert_gmail_draft_baseline(emulate_url: str) -> None: - assert not await _drafts(emulate_url), "seeded provider unexpectedly has drafts" - - -async def _assert_gmail_draft_outcome(emulate_url: str, preview: dict) -> None: - drafts = await _drafts(emulate_url) - assert len(drafts) == 1, drafts - async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client: - response = await client.get( - f"{emulate_url}/gmail/v1/users/me/drafts/{drafts[0]['id']}", - params={"format": "full"}, - ) - response.raise_for_status() - draft = response.json() - assert gmail_header(draft["message"], "Subject") == "REBORN_PROVIDER_CASE_DRAFT" - assert "REBORN_PROVIDER_CASE_DRAFT" in json.dumps(preview), preview - - -async def _gmail_message(emulate_url: str, message_id: str) -> dict: - async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client: - response = await client.get( - f"{emulate_url}/gmail/v1/users/me/messages/{message_id}", - params={"format": "full"}, - ) - response.raise_for_status() - return response.json() - - -async def _assert_gmail_trash_baseline(emulate_url: str) -> None: - message = await _gmail_message(emulate_url, "msg_emulate_unread") - assert "TRASH" not in message["labelIds"], message - - -async def _assert_gmail_trash_outcome(emulate_url: str, preview: dict) -> None: - message = await _gmail_message(emulate_url, "msg_emulate_unread") - assert "TRASH" in message["labelIds"], message - assert "msg_emulate_unread" in json.dumps(preview), preview - - -PROVIDER_OPERATION_CASES = ( - ProviderOperationCase( - case_id="google_drive_get_file", - provider_service="google", - capability_id="google-drive.get_file", - arguments={"file_id": "drv_reborn_qa_brief"}, - assert_baseline=_assert_drive_get_baseline, - assert_outcome=_assert_drive_get_outcome, - ), - ProviderOperationCase( - case_id="gmail_create_draft", - provider_service="google", - capability_id="gmail.create_draft", - arguments={ - "draft": { - "message": { - "raw": raw_mime( - to="draft-recipient@example.com", - subject="REBORN_PROVIDER_CASE_DRAFT", - body="Created through the reusable provider operation runner.", - ) - } - } - }, - assert_baseline=_assert_gmail_draft_baseline, - assert_outcome=_assert_gmail_draft_outcome, - ), - ProviderOperationCase( - case_id="google_drive_update_file", - provider_service="google", - capability_id="google-drive.update_file", - arguments={ - "file_id": "drv_reborn_qa_brief", - "name": "REBORN_PROVIDER_CASE_UPDATED_FILE", - }, - assert_baseline=_assert_drive_update_baseline, - assert_outcome=_assert_drive_update_outcome, - ), - ProviderOperationCase( - case_id="gmail_trash_message", - provider_service="google", - capability_id="gmail.trash_message", - arguments={"message_id": "msg_emulate_unread"}, - assert_baseline=_assert_gmail_trash_baseline, - assert_outcome=_assert_gmail_trash_outcome, - ), -) +PROVIDER_OPERATION_CASES = GOOGLE_PROVIDER_OPERATION_CASES diff --git a/tests/e2e/provider_operation_google_cases.py b/tests/e2e/provider_operation_google_cases.py new file mode 100644 index 00000000000..e78bce10c08 --- /dev/null +++ b/tests/e2e/provider_operation_google_cases.py @@ -0,0 +1,364 @@ +"""Google provider operation cases and provider-side oracles.""" + +import json + +import httpx + +from emulate_provider import gmail_header, google_headers, raw_mime +from provider_operation_types import ProviderOperationCase + +CALENDAR_CREATE_MARKER = "REBORN_PROVIDER_CASE_CREATED_EVENT" +DRIVE_FOLDER_MARKER = "REBORN_PROVIDER_CASE_CREATED_FOLDER" +DRIVE_UPLOAD_CONTENT = "Uploaded through the reusable provider operation runner." +DRIVE_UPLOAD_MARKER = "REBORN_PROVIDER_CASE_UPLOADED_FILE.txt" +GMAIL_REPLY_MARKER = "REBORN_PROVIDER_CASE_REPLY" +SEEDED_GMAIL_MESSAGE_ID = "" +SEEDED_GMAIL_SUBJECT = "Emulate seeded unread" +SEEDED_GMAIL_THREAD_ID = "thr_emulate_unread" + + +async def _get( + emulate_url: str, + path: str, + *, + params: dict[str, str | int] | None = None, +) -> httpx.Response: + async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client: + response = await client.get(f"{emulate_url}{path}", params=params) + response.raise_for_status() + return response + + +async def _drive_file(emulate_url: str, file_id: str) -> dict: + response = await _get(emulate_url, f"/drive/v3/files/{file_id}") + return response.json() + + +async def _drive_files_named(emulate_url: str, name: str) -> list[dict]: + response = await _get( + emulate_url, + "/drive/v3/files", + params={"q": f"name = '{name}' and trashed = false", "pageSize": 100}, + ) + return response.json().get("files", []) + + +async def _assert_drive_get_baseline(emulate_url: str) -> None: + file = await _drive_file(emulate_url, "drv_reborn_qa_brief") + assert file["name"] == "Reborn QA Brief", file + + +async def _assert_drive_get_outcome(emulate_url: str, preview: dict) -> None: + await _assert_drive_get_baseline(emulate_url) + assert "Reborn QA Brief" in json.dumps(preview), preview + + +async def _assert_drive_update_baseline(emulate_url: str) -> None: + await _assert_drive_get_baseline(emulate_url) + + +async def _assert_drive_update_outcome(emulate_url: str, preview: dict) -> None: + file = await _drive_file(emulate_url, "drv_reborn_qa_brief") + assert file["name"] == "REBORN_PROVIDER_CASE_UPDATED_FILE", file + assert "REBORN_PROVIDER_CASE_UPDATED_FILE" in json.dumps(preview), preview + + +async def _assert_drive_folder_baseline(emulate_url: str) -> None: + assert not await _drive_files_named(emulate_url, DRIVE_FOLDER_MARKER) + + +async def _assert_drive_folder_outcome(emulate_url: str, preview: dict) -> None: + matches = await _drive_files_named(emulate_url, DRIVE_FOLDER_MARKER) + assert len(matches) == 1, matches + assert matches[0]["mimeType"] == "application/vnd.google-apps.folder", matches[0] + assert matches[0]["parents"] == ["root"], matches[0] + assert DRIVE_FOLDER_MARKER in json.dumps(preview), preview + + +async def _assert_drive_upload_baseline(emulate_url: str) -> None: + assert not await _drive_files_named(emulate_url, DRIVE_UPLOAD_MARKER) + + +async def _assert_drive_upload_outcome(emulate_url: str, preview: dict) -> None: + matches = await _drive_files_named(emulate_url, DRIVE_UPLOAD_MARKER) + assert len(matches) == 1, matches + uploaded = matches[0] + assert uploaded["mimeType"] == "text/plain", uploaded + assert uploaded["parents"] == ["root"], uploaded + assert uploaded["size"] == str(len(DRIVE_UPLOAD_CONTENT.encode())), uploaded + + response = await _get( + emulate_url, + f"/drive/v3/files/{uploaded['id']}", + params={"alt": "media"}, + ) + assert response.text == DRIVE_UPLOAD_CONTENT + assert DRIVE_UPLOAD_MARKER in json.dumps(preview), preview + + +async def _drafts(emulate_url: str) -> list[dict]: + response = await _get( + emulate_url, + "/gmail/v1/users/me/drafts", + params={"maxResults": 100}, + ) + return response.json().get("drafts", []) + + +async def _assert_gmail_draft_baseline(emulate_url: str) -> None: + assert not await _drafts(emulate_url), "seeded provider unexpectedly has drafts" + + +async def _assert_gmail_draft_outcome(emulate_url: str, preview: dict) -> None: + drafts = await _drafts(emulate_url) + assert len(drafts) == 1, drafts + response = await _get( + emulate_url, + f"/gmail/v1/users/me/drafts/{drafts[0]['id']}", + params={"format": "full"}, + ) + draft = response.json() + assert gmail_header(draft["message"], "Subject") == "REBORN_PROVIDER_CASE_DRAFT" + assert "REBORN_PROVIDER_CASE_DRAFT" in json.dumps(preview), preview + + +async def _gmail_message(emulate_url: str, message_id: str) -> dict: + response = await _get( + emulate_url, + f"/gmail/v1/users/me/messages/{message_id}", + params={"format": "full"}, + ) + return response.json() + + +async def _gmail_thread_messages(emulate_url: str, thread_id: str) -> list[dict]: + response = await _get( + emulate_url, + "/gmail/v1/users/me/messages", + params={"includeSpamTrash": "true", "maxResults": 100}, + ) + messages = [ + message + for message in response.json().get("messages", []) + if message["threadId"] == thread_id + ] + return [ + await _gmail_message(emulate_url, message["id"]) for message in messages + ] + + +async def _assert_gmail_reply_baseline(emulate_url: str) -> None: + seeded = await _gmail_message(emulate_url, "msg_emulate_unread") + assert seeded["threadId"] == SEEDED_GMAIL_THREAD_ID, seeded + assert gmail_header(seeded, "Message-ID") == SEEDED_GMAIL_MESSAGE_ID, seeded + assert gmail_header(seeded, "Subject") == SEEDED_GMAIL_SUBJECT, seeded + thread_messages = await _gmail_thread_messages( + emulate_url, SEEDED_GMAIL_THREAD_ID + ) + assert [message["id"] for message in thread_messages] == [ + "msg_emulate_unread" + ], thread_messages + + +async def _assert_gmail_reply_outcome(emulate_url: str, preview: dict) -> None: + thread_messages = await _gmail_thread_messages( + emulate_url, SEEDED_GMAIL_THREAD_ID + ) + assert len(thread_messages) == 2, thread_messages + assert any( + message["id"] == "msg_emulate_unread" for message in thread_messages + ), thread_messages + replies = [ + message for message in thread_messages if "SENT" in message["labelIds"] + ] + assert len(replies) == 1, thread_messages + reply = replies[0] + assert reply["threadId"] == SEEDED_GMAIL_THREAD_ID, reply + assert gmail_header(reply, "To") == "qa-sender@example.com", reply + assert gmail_header(reply, "Subject") == SEEDED_GMAIL_SUBJECT, reply + assert gmail_header(reply, "In-Reply-To") == SEEDED_GMAIL_MESSAGE_ID, reply + assert gmail_header(reply, "References") == SEEDED_GMAIL_MESSAGE_ID, reply + assert GMAIL_REPLY_MARKER in reply["snippet"], reply + assert GMAIL_REPLY_MARKER in json.dumps(preview), preview + + +async def _assert_gmail_trash_baseline(emulate_url: str) -> None: + message = await _gmail_message(emulate_url, "msg_emulate_unread") + assert "TRASH" not in message["labelIds"], message + + +async def _assert_gmail_trash_outcome(emulate_url: str, preview: dict) -> None: + message = await _gmail_message(emulate_url, "msg_emulate_unread") + assert "TRASH" in message["labelIds"], message + assert "msg_emulate_unread" in json.dumps(preview), preview + + +async def _calendar_events(emulate_url: str, query: str | None = None) -> list[dict]: + params: dict[str, str | int] = {"maxResults": 100} + if query is not None: + params["q"] = query + response = await _get( + emulate_url, + "/calendar/v3/calendars/primary/events", + params=params, + ) + return response.json().get("items", []) + + +async def _assert_calendar_create_baseline(emulate_url: str) -> None: + assert not await _calendar_events(emulate_url, CALENDAR_CREATE_MARKER) + + +async def _assert_calendar_create_outcome(emulate_url: str, preview: dict) -> None: + matches = await _calendar_events(emulate_url, CALENDAR_CREATE_MARKER) + assert len(matches) == 1, matches + event = matches[0] + assert event["summary"] == CALENDAR_CREATE_MARKER, event + assert event["description"] == "Created by the provider operation runner.", event + assert event["start"]["dateTime"] == "2026-07-30T09:00:00.000Z", event + assert event["end"]["dateTime"] == "2026-07-30T09:30:00.000Z", event + assert [attendee["email"] for attendee in event["attendees"]] == [ + "teammate@example.com" + ], event + assert CALENDAR_CREATE_MARKER in json.dumps(preview), preview + + +async def _assert_calendar_delete_baseline(emulate_url: str) -> None: + matching_ids = { + event["id"] for event in await _calendar_events(emulate_url) + } + assert "evt_reborn_planning_sync" in matching_ids, matching_ids + + +async def _assert_calendar_delete_outcome(emulate_url: str, preview: dict) -> None: + matching_ids = { + event["id"] for event in await _calendar_events(emulate_url) + } + assert "evt_reborn_planning_sync" not in matching_ids, matching_ids + assert "evt_reborn_planning_sync" in json.dumps(preview), preview + + +GOOGLE_PROVIDER_OPERATION_CASES = ( + ProviderOperationCase( + case_id="google_drive_get_file", + provider_service="google", + capability_id="google-drive.get_file", + arguments={"file_id": "drv_reborn_qa_brief"}, + assert_baseline=_assert_drive_get_baseline, + assert_outcome=_assert_drive_get_outcome, + ), + ProviderOperationCase( + case_id="gmail_create_draft", + provider_service="google", + capability_id="gmail.create_draft", + arguments={ + "draft": { + "message": { + "raw": raw_mime( + to="draft-recipient@example.com", + subject="REBORN_PROVIDER_CASE_DRAFT", + body="Created through the reusable provider operation runner.", + ) + } + } + }, + assert_baseline=_assert_gmail_draft_baseline, + assert_outcome=_assert_gmail_draft_outcome, + ), + ProviderOperationCase( + case_id="google_drive_update_file", + provider_service="google", + capability_id="google-drive.update_file", + arguments={ + "file_id": "drv_reborn_qa_brief", + "name": "REBORN_PROVIDER_CASE_UPDATED_FILE", + }, + assert_baseline=_assert_drive_update_baseline, + assert_outcome=_assert_drive_update_outcome, + ), + ProviderOperationCase( + case_id="gmail_trash_message", + provider_service="google", + capability_id="gmail.trash_message", + arguments={"message_id": "msg_emulate_unread"}, + assert_baseline=_assert_gmail_trash_baseline, + assert_outcome=_assert_gmail_trash_outcome, + ), + ProviderOperationCase( + case_id="gmail_reply_to_message", + provider_service="google", + capability_id="gmail.reply_to_message", + arguments={ + "message": { + "raw": raw_mime( + to="qa-sender@example.com", + subject=SEEDED_GMAIL_SUBJECT, + body=( + f"{GMAIL_REPLY_MARKER}: Reply sent through the reusable " + "provider operation runner." + ), + in_reply_to=SEEDED_GMAIL_MESSAGE_ID, + references=SEEDED_GMAIL_MESSAGE_ID, + ), + "threadId": SEEDED_GMAIL_THREAD_ID, + } + }, + assert_baseline=_assert_gmail_reply_baseline, + assert_outcome=_assert_gmail_reply_outcome, + ), + ProviderOperationCase( + case_id="google_calendar_create_event", + provider_service="google", + capability_id="google-calendar.create_event", + arguments={ + "calendar_id": "primary", + "event": { + "summary": CALENDAR_CREATE_MARKER, + "description": "Created by the provider operation runner.", + "start": { + "dateTime": "2026-07-30T09:00:00.000Z", + "timeZone": "UTC", + }, + "end": { + "dateTime": "2026-07-30T09:30:00.000Z", + "timeZone": "UTC", + }, + "attendees": [{"email": "teammate@example.com"}], + }, + }, + assert_baseline=_assert_calendar_create_baseline, + assert_outcome=_assert_calendar_create_outcome, + ), + ProviderOperationCase( + case_id="google_calendar_delete_event", + provider_service="google", + capability_id="google-calendar.delete_event", + arguments={ + "calendar_id": "primary", + "event_id": "evt_reborn_planning_sync", + }, + assert_baseline=_assert_calendar_delete_baseline, + assert_outcome=_assert_calendar_delete_outcome, + ), + ProviderOperationCase( + case_id="google_drive_create_folder", + provider_service="google", + capability_id="google-drive.create_folder", + arguments={"name": DRIVE_FOLDER_MARKER, "parent_id": "root"}, + assert_baseline=_assert_drive_folder_baseline, + assert_outcome=_assert_drive_folder_outcome, + ), + ProviderOperationCase( + case_id="google_drive_upload_file", + provider_service="google", + capability_id="google-drive.upload_file", + arguments={ + "name": DRIVE_UPLOAD_MARKER, + "content": DRIVE_UPLOAD_CONTENT, + "mime_type": "text/plain", + "parent_id": "root", + }, + assert_baseline=_assert_drive_upload_baseline, + assert_outcome=_assert_drive_upload_outcome, + ), +) diff --git a/tests/e2e/provider_operation_types.py b/tests/e2e/provider_operation_types.py new file mode 100644 index 00000000000..5bba773226f --- /dev/null +++ b/tests/e2e/provider_operation_types.py @@ -0,0 +1,21 @@ +"""Shared types for full-path provider operation cases.""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Literal + +BaselineAssertion = Callable[[str], Awaitable[None]] +OutcomeAssertion = Callable[[str, dict], Awaitable[None]] +ProviderService = Literal["google", "github", "slack"] + + +@dataclass(frozen=True) +class ProviderOperationCase: + """One capability invocation with provider-observable proof.""" + + case_id: str + provider_service: ProviderService + capability_id: str + arguments: dict + assert_baseline: BaselineAssertion + assert_outcome: OutcomeAssertion