Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions tests/e2e/emulate_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,21 @@ def gmail_header(message: dict, name: str) -> str | None:
return None


def raw_mime(*, to: str, subject: str, body: str) -> str:
message = (
f"To: {to}\r\n"
f"Subject: {subject}\r\n"
"Content-Type: text/plain; charset=utf-8\r\n"
"\r\n"
f"{body}"
)
def raw_mime(
*,
to: str,
subject: str,
body: str,
in_reply_to: str | None = None,
references: str | None = None,
) -> str:
headers = [f"To: {to}", f"Subject: {subject}"]
if in_reply_to:
headers.append(f"In-Reply-To: {in_reply_to}")
if references:
headers.append(f"References: {references}")
headers.append("Content-Type: text/plain; charset=utf-8")
message = "\r\n".join([*headers, "", body])
return base64.urlsafe_b64encode(message.encode("utf-8")).decode("ascii").rstrip("=")


Expand Down
2 changes: 2 additions & 0 deletions tests/e2e/fixtures/emulate/google_gmail.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ google:
name: Secondary Google User
messages:
- id: msg_emulate_unread
thread_id: thr_emulate_unread
message_id: "<msg_emulate_unread@ironclaw.test>"
user_email: e2e.google@example.com
from: qa-sender@example.com
to: e2e.google@example.com
Expand Down
10 changes: 5 additions & 5 deletions tests/e2e/fixtures/provider_capability_coverage.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,22 @@ tested = [
"gmail.create_draft",
"gmail.get_message",
"gmail.list_messages",
"gmail.reply_to_message",
"gmail.send_message",
"gmail.trash_message",
"google-calendar.create_event",
"google-calendar.delete_event",
"google-calendar.list_calendars",
"google-calendar.list_events",
"google-docs.create_document",
"google-docs.insert_text",
"google-docs.read_content",
"google-drive.create_folder",
"google-drive.download_file",
"google-drive.get_file",
"google-drive.list_files",
"google-drive.update_file",
"google-drive.upload_file",
"google-sheets.append_values",
"google-sheets.create_spreadsheet",
"google-sheets.get_spreadsheet",
Expand Down Expand Up @@ -92,10 +97,7 @@ capabilities = [
"github.unresolve_review_thread",
"github.update_issue",
"github.update_pull_request",
"gmail.reply_to_message",
"google-calendar.add_attendees",
"google-calendar.create_event",
"google-calendar.delete_event",
"google-calendar.find_free_slots",
"google-calendar.get_event",
"google-calendar.set_reminder",
Expand All @@ -108,14 +110,12 @@ capabilities = [
"google-docs.get_document",
"google-docs.insert_table",
"google-docs.replace_text",
"google-drive.create_folder",
"google-drive.delete_file",
"google-drive.list_permissions",
"google-drive.list_shared_drives",
"google-drive.remove_permission",
"google-drive.share_file",
"google-drive.trash_file",
"google-drive.upload_file",
"google-sheets.add_sheet",
"google-sheets.batch_read_values",
"google-sheets.clear_values",
Expand Down
153 changes: 4 additions & 149 deletions tests/e2e/provider_operation_cases.py
Original file line number Diff line number Diff line change
@@ -1,151 +1,6 @@
"""Typed full-path provider operation cases and provider-side oracles."""
"""Registry for typed full-path provider operation cases."""

from collections.abc import Awaitable, Callable
from dataclasses import dataclass
import json
from typing import Literal
from provider_operation_google_cases import GOOGLE_PROVIDER_OPERATION_CASES
from provider_operation_types import ProviderOperationCase

import httpx

from emulate_provider import gmail_header, google_headers, raw_mime

BaselineAssertion = Callable[[str], Awaitable[None]]
OutcomeAssertion = Callable[[str, dict], Awaitable[None]]
ProviderService = Literal["google", "github", "slack"]


@dataclass(frozen=True)
class ProviderOperationCase:
"""One capability invocation with provider-observable proof."""

case_id: str
provider_service: ProviderService
capability_id: str
arguments: dict
assert_baseline: BaselineAssertion
assert_outcome: OutcomeAssertion


async def _drive_file(emulate_url: str, file_id: str) -> dict:
async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client:
response = await client.get(f"{emulate_url}/drive/v3/files/{file_id}")
response.raise_for_status()
return response.json()


async def _assert_drive_get_baseline(emulate_url: str) -> None:
file = await _drive_file(emulate_url, "drv_reborn_qa_brief")
assert file["name"] == "Reborn QA Brief", file


async def _assert_drive_get_outcome(emulate_url: str, preview: dict) -> None:
await _assert_drive_get_baseline(emulate_url)
assert "Reborn QA Brief" in json.dumps(preview), preview


async def _assert_drive_update_baseline(emulate_url: str) -> None:
await _assert_drive_get_baseline(emulate_url)


async def _assert_drive_update_outcome(emulate_url: str, preview: dict) -> None:
file = await _drive_file(emulate_url, "drv_reborn_qa_brief")
assert file["name"] == "REBORN_PROVIDER_CASE_UPDATED_FILE", file
assert "REBORN_PROVIDER_CASE_UPDATED_FILE" in json.dumps(preview), preview


async def _drafts(emulate_url: str) -> list[dict]:
async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client:
response = await client.get(
f"{emulate_url}/gmail/v1/users/me/drafts",
params={"maxResults": 100},
)
response.raise_for_status()
return response.json().get("drafts", [])


async def _assert_gmail_draft_baseline(emulate_url: str) -> None:
assert not await _drafts(emulate_url), "seeded provider unexpectedly has drafts"


async def _assert_gmail_draft_outcome(emulate_url: str, preview: dict) -> None:
drafts = await _drafts(emulate_url)
assert len(drafts) == 1, drafts
async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client:
response = await client.get(
f"{emulate_url}/gmail/v1/users/me/drafts/{drafts[0]['id']}",
params={"format": "full"},
)
response.raise_for_status()
draft = response.json()
assert gmail_header(draft["message"], "Subject") == "REBORN_PROVIDER_CASE_DRAFT"
assert "REBORN_PROVIDER_CASE_DRAFT" in json.dumps(preview), preview


async def _gmail_message(emulate_url: str, message_id: str) -> dict:
async with httpx.AsyncClient(headers=google_headers(), timeout=15) as client:
response = await client.get(
f"{emulate_url}/gmail/v1/users/me/messages/{message_id}",
params={"format": "full"},
)
response.raise_for_status()
return response.json()


async def _assert_gmail_trash_baseline(emulate_url: str) -> None:
message = await _gmail_message(emulate_url, "msg_emulate_unread")
assert "TRASH" not in message["labelIds"], message


async def _assert_gmail_trash_outcome(emulate_url: str, preview: dict) -> None:
message = await _gmail_message(emulate_url, "msg_emulate_unread")
assert "TRASH" in message["labelIds"], message
assert "msg_emulate_unread" in json.dumps(preview), preview


PROVIDER_OPERATION_CASES = (
ProviderOperationCase(
case_id="google_drive_get_file",
provider_service="google",
capability_id="google-drive.get_file",
arguments={"file_id": "drv_reborn_qa_brief"},
assert_baseline=_assert_drive_get_baseline,
assert_outcome=_assert_drive_get_outcome,
),
ProviderOperationCase(
case_id="gmail_create_draft",
provider_service="google",
capability_id="gmail.create_draft",
arguments={
"draft": {
"message": {
"raw": raw_mime(
to="draft-recipient@example.com",
subject="REBORN_PROVIDER_CASE_DRAFT",
body="Created through the reusable provider operation runner.",
)
}
}
},
assert_baseline=_assert_gmail_draft_baseline,
assert_outcome=_assert_gmail_draft_outcome,
),
ProviderOperationCase(
case_id="google_drive_update_file",
provider_service="google",
capability_id="google-drive.update_file",
arguments={
"file_id": "drv_reborn_qa_brief",
"name": "REBORN_PROVIDER_CASE_UPDATED_FILE",
},
assert_baseline=_assert_drive_update_baseline,
assert_outcome=_assert_drive_update_outcome,
),
ProviderOperationCase(
case_id="gmail_trash_message",
provider_service="google",
capability_id="gmail.trash_message",
arguments={"message_id": "msg_emulate_unread"},
assert_baseline=_assert_gmail_trash_baseline,
assert_outcome=_assert_gmail_trash_outcome,
),
)
PROVIDER_OPERATION_CASES = GOOGLE_PROVIDER_OPERATION_CASES
Loading
Loading