From 8a3fd5f1adfb90e114e29c82d914ca7d0674da8c Mon Sep 17 00:00:00 2001 From: Luke Maj Date: Tue, 20 Oct 2026 06:32:47 +0200 Subject: [PATCH 1/3] wip: combine qualified resources with destination automation --- .../BrowserPreviewRouteScreen.test.tsx | 156 + .../browser/BrowserPreviewRouteScreen.tsx | 183 +- .../browser/PreviewStreamWebView.test.tsx | 522 ++ .../features/browser/PreviewStreamWebView.tsx | 117 +- .../browser/preview-stream-document.test.ts | 61 + .../browser/preview-stream-document.ts | 30 +- .../browser/preview-stream.browser.ts | 4 + .../devices/DevicePreviewRouteScreen.tsx | 87 +- .../devices/threadDevicePreviews.test.ts | 9 + .../features/devices/threadDevicePreviews.ts | 6 +- .../features/threads/ThreadDetailScreen.tsx | 112 + apps/mobile/src/state/device.ts | 49 +- apps/mobile/src/state/preview.ts | 42 +- apps/server/src/assets/AssetAccess.ts | 470 + .../src/assets/AttachmentStaging.test.ts | 168 +- apps/server/src/assets/AttachmentStaging.ts | 620 +- .../src/assets/NativeBrowserDownload.test.ts | 98 + apps/server/src/auth/EnvironmentAuth.ts | 31 +- apps/server/src/auth/RpcAuthorization.test.ts | 64 + apps/server/src/auth/SessionStore.test.ts | 238 +- apps/server/src/auth/SessionStore.ts | 310 + apps/server/src/device/DeviceHubProxy.test.ts | 101 +- apps/server/src/device/DeviceHubProxy.ts | 1040 ++- apps/server/src/device/DeviceService.test.ts | 57 +- apps/server/src/device/DeviceService.ts | 155 +- apps/server/src/forkWsRpc.ts | 10 +- apps/server/src/http.ts | 264 +- .../src/http/ResourceWebSocketServer.test.ts | 226 + .../src/http/ResourceWebSocketServer.ts | 109 + apps/server/src/mcp/McpHttpServer.ts | 228 +- apps/server/src/mcp/McpToolAccess.test.ts | 69 + apps/server/src/mcp/McpToolAccess.ts | 63 + .../server/src/mcp/PreviewAutomationBroker.ts | 137 +- .../src/mcp/toolkits/attachment/handlers.ts | 115 +- .../src/mcp/toolkits/attachment/tools.ts | 34 + .../src/mcp/toolkits/device/handlers.ts | 18 +- apps/server/src/mcp/toolkits/device/tools.ts | 8 +- .../src/mcp/toolkits/preview/handlers.ts | 213 +- apps/server/src/mcp/toolkits/preview/tools.ts | 10 +- .../mcp/toolkits/previewControls/handlers.ts | 34 +- .../src/observability/RpcInstrumentation.ts | 4 + apps/server/src/peer/PeerAuthority.ts | 270 +- ...eerAuthority.typedEffects.workflow.test.ts | 7680 ++++++++++++++++- .../peer/PeerAuthority.workflow.fixture.ts | 853 +- apps/server/src/peer/PeerForwarding.ts | 415 +- apps/server/src/peer/PeerMcpClient.ts | 516 ++ apps/server/src/peer/PeerToolRouting.test.ts | 49 +- apps/server/src/preview/Manager.ts | 64 +- apps/server/src/preview/ServerBrowser.ts | 1313 ++- .../src/preview/ServerBrowserPage.test.ts | 99 + apps/server/src/preview/ServerBrowserPage.ts | 152 +- .../src/preview/ServerBrowserStream.test.ts | 50 +- .../server/src/preview/ServerBrowserStream.ts | 873 +- .../server/src/preview/SessionControl.test.ts | 112 + apps/server/src/preview/SessionControl.ts | 64 +- .../testing/BrowserExternal.fixture.ts | 216 + apps/server/src/server.ts | 31 +- apps/server/src/ws.ts | 41 +- apps/web/src/browser/ServerBrowserSurface.tsx | 68 +- .../web/src/browser/serverPictureInPicture.ts | 33 +- .../src/components/chat/MessagesTimeline.tsx | 104 +- .../chat/RightPanelSurfaceContent.tsx | 3 + .../web/src/components/device/DevicePanel.tsx | 161 +- .../device/DeviceStreamView.browser.test.tsx | 14 +- .../components/device/DeviceStreamView.tsx | 55 +- .../src/components/preview/PreviewPanel.tsx | 137 +- .../preview/ThreadPreviewMiniPlayer.tsx | 111 +- apps/web/src/previewMiniPlayerStore.test.ts | 46 + apps/web/src/previewMiniPlayerStore.ts | 11 +- apps/web/src/rightPanelStore.test.ts | 91 +- apps/web/src/rightPanelStore.ts | 86 +- apps/web/src/state/device.ts | 44 +- apps/web/src/state/previewStream.ts | 91 +- docs/fork-features.md | 77 +- docs/map.md | 2 +- docs/user/outside-agents.md | 20 + .../client-runtime/src/device/hubAccess.ts | 7 + .../src/device/screenshot.test.ts | 28 + .../client-runtime/src/device/screenshot.ts | 10 +- .../client-runtime/src/device/stream.test.ts | 196 +- packages/client-runtime/src/device/stream.ts | 74 +- .../src/preview/serverBrowserStream.test.ts | 298 +- .../src/preview/serverBrowserStream.ts | 131 +- .../client-runtime/src/state/assets.test.ts | 139 + packages/client-runtime/src/state/assets.ts | 57 +- packages/client-runtime/src/state/device.ts | 19 +- .../src/state/deviceHubAccess.test.ts | 263 + .../src/state/deviceHubAccess.ts | 136 +- packages/client-runtime/src/state/preview.ts | 19 +- .../client-runtime/src/t3ToolSummary.test.ts | 49 + packages/client-runtime/src/t3ToolSummary.ts | 32 + packages/contracts/src/assets.test.ts | 113 +- packages/contracts/src/assets.ts | 46 + .../contracts/src/clientRpcPermissions.ts | 20 + packages/contracts/src/device.ts | 50 +- packages/contracts/src/environmentHttp.ts | 26 + packages/contracts/src/peerAttachments.ts | 86 +- packages/contracts/src/preview.test.ts | 110 + packages/contracts/src/preview.ts | 129 +- packages/contracts/src/previewAutomation.ts | 42 +- packages/contracts/src/rpc.ts | 44 +- .../shared/src/t3McpToolPresentation.test.ts | 9 + packages/shared/src/t3McpToolPresentation.ts | 10 + packages/shared/src/toolOutput.test.ts | 249 + packages/shared/src/toolOutput.ts | 109 +- scripts/fork-upstream-edits.txt | 57 +- vite.config.ts | 5 + 107 files changed, 21674 insertions(+), 1113 deletions(-) create mode 100644 apps/mobile/src/features/browser/BrowserPreviewRouteScreen.test.tsx create mode 100644 apps/mobile/src/features/browser/PreviewStreamWebView.test.tsx create mode 100644 apps/mobile/src/features/browser/preview-stream-document.test.ts create mode 100644 apps/server/src/assets/NativeBrowserDownload.test.ts create mode 100644 apps/server/src/http/ResourceWebSocketServer.test.ts create mode 100644 apps/server/src/http/ResourceWebSocketServer.ts create mode 100644 apps/server/src/preview/testing/BrowserExternal.fixture.ts create mode 100644 packages/client-runtime/src/state/deviceHubAccess.test.ts diff --git a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.test.tsx b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.test.tsx new file mode 100644 index 0000000000..65ffade0b9 --- /dev/null +++ b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.test.tsx @@ -0,0 +1,156 @@ +// @vitest-environment jsdom +import { act, useEffect, useImperativeHandle, type ComponentProps, type ReactNode } from "react"; +import { createRoot } from "react-dom/client"; +import { EnvironmentId, ThreadId, PreviewTabId } from "@t3tools/contracts"; +import { expect, it, vi } from "vite-plus/test"; +const observed = vi.hoisted(() => ({ command: vi.fn(), alert: vi.fn() })); +vi.mock("react-native", () => ({ + Platform: { OS: "android" }, + AppState: { currentState: "active", addEventListener: () => ({ remove() {} }) }, + Alert: { alert: observed.alert }, + View: ({ children }: { children?: ReactNode }) =>
{children}
, + Pressable: ({ + children, + onPress, + disabled, + }: { + children?: ReactNode; + onPress?: () => void; + disabled?: boolean; + }) => ( + + ), + TextInput: ({ + value, + onChangeText, + onSubmitEditing, + accessibilityLabel, + editable, + }: { + value: string; + onChangeText: (value: string) => void; + onSubmitEditing: () => void; + accessibilityLabel: string; + editable: boolean; + }) => ( + onChangeText(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") onSubmitEditing(); + }} + /> + ), +})); +vi.mock("@react-navigation/native", () => ({ + useIsFocused: () => true, + useNavigation: () => ({ goBack() {} }), +})); +vi.mock("@react-navigation/native-stack", () => ({ + createNativeStackNavigator: () => ({ + Navigator: ({ children }: { children: ReactNode }) => children, + Screen: ({ children }: { children: () => ReactNode }) => children(), + }), +})); +vi.mock("react-native-safe-area-context", () => ({ useSafeAreaInsets: () => ({ bottom: 0 }) })); +vi.mock("../../state/preview", () => ({ useThreadServerBrowserTabs: () => null })); +vi.mock("../settings/appearance/AppearancePreferencesProvider", () => ({ + useAppearancePreferences: () => ({ themeVariables: {} }), +})); +vi.mock("../../components/ui/Toolbar", () => ({ Toolbar: () => null })); +vi.mock("../../components/ui/AppActivityIndicator", () => ({ AppActivityIndicator: () => null })); +vi.mock("../../components/ui/ControlPill", () => ({ ControlPill: () => null })); +vi.mock("../../components/ui/AppBar", () => ({ AppBar: () => null })); +vi.mock("../../components/ui/AppText", () => ({ + AppText: ({ children }: { children?: ReactNode }) => {children}, +})); +vi.mock("./BrowserTabMenu", () => ({ BrowserTabMenu: () => null })); +vi.mock("./PreviewStreamWebView", () => ({ + PreviewStreamWebView: ( + props: ComponentProps, + ) => { + useImperativeHandle( + props.ref, + () => ({ command: observed.command, togglePictureInPicture() {} }), + [], + ); + useEffect(() => { + props.onStreamingChange?.(true); + props.onControl?.({ canOperate: true, controller: "you", generation: 1, dialog: null }); + }, []); + return null; + }, +})); +import { BrowserPreviewRouteScreen } from "./BrowserPreviewRouteScreen"; +it("qualified navigation reports invalid addresses and sends only normalized valid input", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + observed.command.mockClear(); + observed.alert.mockClear(); + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + const errors: unknown[] = []; + const error = (event: ErrorEvent) => { + errors.push(event.error); + event.preventDefault(); + }; + window.addEventListener("error", error); + try { + await act(async () => + root.render( + , + ), + ); + const input = container.querySelector("input")!; + const go = [...container.querySelectorAll("button")].find( + (button) => button.textContent === "Go", + )!; + expect(input.disabled).toBe(false); + const enter = async (value: string) => + act(async () => { + input.value = value; + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + for (const [index, value] of ["http://", "file:///private/test"].entries()) { + await enter(value); + await act(async () => { + if (index === 0) go.click(); + else input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); + }); + expect(observed.command).not.toHaveBeenCalled(); + expect(observed.alert).toHaveBeenLastCalledWith( + "Could not open this address", + "Enter an http or https URL.", + ); + } + expect(errors).toEqual([]); + await enter("localhost:3000"); + await act(async () => go.click()); + expect(observed.command.mock.calls).toEqual([ + [{ type: "navigate", url: "http://localhost:3000/" }], + ]); + } finally { + await act(async () => root.unmount()); + window.removeEventListener("error", error); + container.remove(); + vi.unstubAllGlobals(); + } +}); diff --git a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx index 7598dc1421..6912045a20 100644 --- a/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx +++ b/apps/mobile/src/features/browser/BrowserPreviewRouteScreen.tsx @@ -4,19 +4,20 @@ import * as Dialog from "../../components/ui/Dialog"; import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; import { createNativeStackNavigator } from "@react-navigation/native-stack"; import { + PreviewViewerResource, EnvironmentId, - FILL_PREVIEW_VIEWPORT, ThreadId, type PreviewSessionSnapshot, } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; import { useCallback, useEffect, useRef, useState } from "react"; -import { AppState, Platform, TextInput, View } from "react-native"; +import { AppState, Platform, TextInput, View, Pressable } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; - +import { AppText } from "../../components/ui/AppText"; import { ControlPill } from "../../components/ui/ControlPill"; import { AppBar } from "../../components/ui/AppBar"; - import { useThreadServerBrowserTabs } from "../../state/preview"; import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; import { BrowserTabMenu } from "./BrowserTabMenu"; @@ -35,11 +36,20 @@ type BrowserPreviewRouteScreenProps = StaticScreenProps<{ readonly environmentId: string; readonly threadId: string; readonly tabId?: string; + readonly resource?: PreviewViewerResource; }>; export function BrowserPreviewRouteScreen({ route }: BrowserPreviewRouteScreenProps) { const navigation = useNavigation(); const onClose = useCallback(() => navigation.goBack(), [navigation]); + const resource = + route.params.resource === undefined + ? undefined + : Option.getOrNull(Schema.decodeUnknownOption(PreviewViewerResource)(route.params.resource)); + useEffect(() => { + if (resource === null) onClose(); + }, [resource, onClose]); + if (resource === null) return null; return ( - {() => ( - - )} + {() => + resource === undefined ? ( + + ) : ( + + ) + } ); } +function QualifiedBrowserPreviewScreen({ + environmentId, + resource, + onClose, +}: { + readonly environmentId: EnvironmentId; + readonly resource: PreviewViewerResource; + readonly onClose: () => void; +}) { + const focused = useIsFocused(); + const { themeVariables } = useAppearancePreferences(); + const [foreground, setForeground] = useState(AppState.currentState !== "background"); + const [selected, setSelected] = useState(resource); + const [tabs, setTabs] = useState([resource]); + const [address, setAddress] = useState(""); + const [canControl, setCanControl] = useState(false); + const [streaming, setStreaming] = useState(false); + const [pictureInPicture, setPictureInPicture] = useState(NO_PICTURE_IN_PICTURE); + const streamRef = useRef(null); + useEffect(() => { + const subscription = AppState.addEventListener("change", (state) => + setForeground(state !== "background"), + ); + return () => subscription.remove(); + }, []); + useEffect(() => { + setSelected(resource); + setTabs([resource]); + }, [resource]); + const navigate = () => { + if (!canControl || !streaming || !address.trim()) return; + const stream = streamRef.current; + if (stream === null) return; + try { + stream.command({ type: "navigate", url: normalizePreviewUrl(address) }); + } catch { + Dialog.alert("Could not open this address", "Enter an http or https URL."); + } + }; + return ( + + streamRef.current?.togglePictureInPicture(), + }, + ] + : [] + } + menus={ + tabs.length > 1 + ? [ + { + title: "Tabs", + icon: "square.on.square", + items: tabs.map((tab, index) => ({ + id: `tab-${index}`, + title: `Browser ${index + 1}`, + selected: tab.tabId === selected.tabId, + onPress: () => { + setSelected(tab); + setPictureInPicture(NO_PICTURE_IN_PICTURE); + }, + })), + }, + ] + : [] + } + /> + + + Close view + + streamRef.current?.command({ type: "history", delta: -1 })} + > + Back + + + + Go + + + { + setPictureInPicture(state); + if (detail) Dialog.alert("Picture in picture is unavailable", detail); + }} + onStreamingChange={setStreaming} + onControl={(control) => setCanControl(control?.controller === "you")} + onPopup={(popup) => { + setTabs((current) => + current.some((tab) => tab.tabId === popup.tabId) ? current : [...current, popup], + ); + setSelected(popup); + setPictureInPicture(NO_PICTURE_IN_PICTURE); + }} + /> + + ); +} + function BrowserPreviewScreen({ environmentId, threadId, diff --git a/apps/mobile/src/features/browser/PreviewStreamWebView.test.tsx b/apps/mobile/src/features/browser/PreviewStreamWebView.test.tsx new file mode 100644 index 0000000000..c6908c094e --- /dev/null +++ b/apps/mobile/src/features/browser/PreviewStreamWebView.test.tsx @@ -0,0 +1,522 @@ +// @vitest-environment jsdom +import { act, useEffect, type ReactNode } from "react"; +import { createRoot } from "react-dom/client"; +import type { WebViewMessageEvent, WebViewProps } from "react-native-webview"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { expect, it, vi } from "vite-plus/test"; +const observed = vi.hoisted(() => ({ + message: undefined as WebViewProps["onMessage"], + mounts: 0, + unmounts: 0, + appState: "active", + focused: true, + listeners: new Set<(state: string) => void>(), + renew: vi.fn(), + refresh: vi.fn(), + picker: vi.fn(), + share: vi.fn(), + alert: vi.fn(), + changePrompt: undefined as ((value: string) => void) | undefined, +})); +vi.mock("react-native", () => ({ + Platform: { OS: "android" }, + AppState: { + get currentState() { + return observed.appState; + }, + addEventListener: (_: string, listener: (state: string) => void) => { + observed.listeners.add(listener); + return { remove: () => observed.listeners.delete(listener) }; + }, + }, + View: ({ children }: { children?: ReactNode }) =>
{children}
, + Pressable: ({ children, onPress }: { children?: ReactNode; onPress?: () => void }) => ( + + ), + TextInput: ({ + value, + onChangeText, + }: { + value: string; + onChangeText: (value: string) => void; + }) => { + observed.changePrompt = onChangeText; + return ; + }, +})); +vi.mock("react-native-webview", () => ({ + WebView: (props: WebViewProps) => { + observed.message = props.onMessage; + useEffect(() => { + observed.mounts++; + return () => { + observed.unmounts++; + }; + }, []); + return
; + }, +})); +vi.mock("@t3tools/mobile-preview-stream", () => ({ default: "" })); +vi.mock("expo-clipboard", () => ({ setStringAsync: async () => {} })); +vi.mock("expo-document-picker", () => ({ getDocumentAsync: observed.picker })); +vi.mock("../../components/ui/Dialog", () => ({ alert: observed.alert })); +vi.mock("../../components/ui/AppActivityIndicator", () => ({ AppActivityIndicator: () => null })); +vi.mock("../../components/ui/AppText", () => ({ + AppText: ({ children }: { children?: ReactNode }) => {children}, +})); +vi.mock("../../lib/attachmentDownload", () => ({ downloadAndShareAttachment: observed.share })); +vi.mock("../../state/preview", () => { + const access = { + httpBase: "https://a.test/api/preview-stream", + wsBase: "wss://a.test/api/preview-stream", + query: { viewerTicket: "expired" }, + credentials: false, + canOperate: true, + requestCurrentAccess: observed.renew, + }; + return { + usePreviewStreamAccess: () => ({ access, error: null, refresh: observed.refresh }), + useThreadServerBrowserTabs: () => null, + }; +}); +vi.mock("@react-navigation/native", () => ({ + useIsFocused: () => observed.focused, + useNavigation: () => ({ goBack() {} }), +})); +vi.mock("@react-navigation/native-stack", () => ({ + createNativeStackNavigator: () => ({ + Navigator: ({ children }: { children: ReactNode }) => children, + Screen: ({ children }: { children: () => ReactNode }) => children(), + }), +})); +vi.mock("react-native-safe-area-context", () => ({ useSafeAreaInsets: () => ({ bottom: 0 }) })); +vi.mock("../settings/appearance/AppearancePreferencesProvider", () => ({ + useAppearancePreferences: () => ({ themeVariables: {} }), +})); +vi.mock("../../components/ui/Toolbar", () => ({ Toolbar: () => null })); +vi.mock("../../components/ui/ControlPill", () => ({ ControlPill: () => null })); +vi.mock("../../components/ui/AppBar", () => ({ AppBar: () => null })); +vi.mock("./BrowserTabMenu", () => ({ BrowserTabMenu: () => null })); +import { BrowserPreviewRouteScreen } from "./BrowserPreviewRouteScreen"; +import { isForegroundHandoffActive } from "../../lib/foreground-handoff"; +import { PreviewStreamWebView } from "./PreviewStreamWebView"; + +it("admits native files after picker completion and downloads after save confirmation without reopening control", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + let ticket = "not-yet-current"; + let retired = false; + observed.renew.mockImplementation(async () => { + if (retired) throw new Error("Original authority retired"); + return { query: { viewerTicket: ticket } }; + }); + const chosen = Promise.withResolvers<{ + canceled: false; + assets: Array<{ uri: string; name: string; mimeType: string }>; + }>(); + observed.picker.mockReturnValue(chosen.promise); + const uploaded = Promise.withResolvers(); + const posted: Array<{ url: string; body: unknown }> = []; + vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { + posted.push({ url, body: init.body }); + uploaded.resolve(); + return new Response(null, { status: 204 }); + }); + class NativeFormData { + readonly parts: Array<{ name: string; value: unknown }> = []; + append(name: string, value: unknown) { + this.parts.push({ name, value }); + } + } + vi.stubGlobal("FormData", NativeFormData); + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + const message = (value: unknown) => + observed.message!({ nativeEvent: { data: JSON.stringify(value) } } as WebViewMessageEvent); + const picker = { + multiple: false, + accept: "", + uploadUrl: + "https://a.test/api/preview-stream/upload?threadId=b&tabId=tab&chooser=exact&viewerTicket=expired", + credentials: false, + }; + const click = (text: string) => { + const button = [...host.querySelectorAll("button")].find((value) => value.textContent === text); + expect(button).toBeDefined(); + button!.click(); + }; + try { + await act(async () => + root.render( + , + ), + ); + await act(async () => + message({ + type: "control", + controller: "you", + canOperate: true, + generation: 3, + dialog: { type: "prompt", message: "Enter a response", defaultValue: "original" }, + }), + ); + const draft = "界".repeat(16385); + await act(async () => observed.changePrompt!(draft)); + expect(host.querySelector("input")?.value).toBe(draft); + const mountsBeforeRefusal = observed.mounts; + await act(async () => + message({ + type: "inputRejected", + message: "Browser input is invalid or too large. Edit it and try again.", + }), + ); + expect(observed.alert).toHaveBeenCalledWith( + "Browser input refused", + "Browser input is invalid or too large. Edit it and try again.", + ); + expect(observed.mounts).toBe(mountsBeforeRefusal); + expect(host.querySelector("input")?.value).toBe(draft); + expect(host.textContent).not.toContain("Reconnect"); + await act(async () => { + message({ + type: "control", + controller: "you", + canOperate: true, + generation: 3, + dialog: null, + }); + message({ type: "fileChooser", chooser: picker }); + }); + await act(async () => click("Choose file")); + expect(observed.renew).not.toHaveBeenCalled(); + ticket = "fresh-after-native-picker"; + await act(async () => { + chosen.resolve({ + canceled: false, + assets: [{ uri: "file:///fixture/chosen.txt", name: "chosen.txt", mimeType: "text/plain" }], + }); + await uploaded.promise; + }); + expect(new URL(posted[0]!.url).searchParams.get("viewerTicket")).toBe(ticket); + expect((posted[0]!.body as NativeFormData).parts).toEqual([ + { + name: "file", + value: { uri: "file:///fixture/chosen.txt", name: "chosen.txt", type: "text/plain" }, + }, + ]); + ticket = "fresh-cancel"; + await act(async () => { + message({ type: "fileChooser", chooser: picker }); + }); + await act(async () => click("Cancel")); + expect(new URL(posted[1]!.url).searchParams.get("viewerTicket")).toBe(ticket); + expect((posted[1]!.body as NativeFormData).parts).toEqual([]); + await act(async () => + message({ + type: "download", + fileName: "report.txt", + sizeBytes: 4, + url: "https://a.test/api/preview-stream/download?threadId=b&tabId=tab&id=exact-download&viewerTicket=expired", + }), + ); + const buttons = observed.alert.mock.calls.at(-1)![2] as Array<{ + text: string; + onPress?: () => void; + }>; + ticket = "fresh-save"; + const shared = Promise.withResolvers(); + observed.share.mockImplementation(async ({ url }: { url: string }) => { + shared.resolve(url); + }); + await act(async () => { + buttons.find((button) => button.text === "Save or share")!.onPress!(); + expect(new URL(await shared.promise).searchParams.get("viewerTicket")).toBe(ticket); + }); + retired = true; + await act(async () => message({ type: "fileChooser", chooser: picker })); + await act(async () => click("Cancel")); + expect(observed.alert).toHaveBeenLastCalledWith( + "Could not send the files to the page", + "Original authority retired", + ); + expect(posted).toHaveLength(2); + expect(observed.share).toHaveBeenCalledTimes(1); + expect(observed.mounts).toBe(1); + expect(observed.refresh).not.toHaveBeenCalled(); + } finally { + await act(async () => root.unmount()); + host.remove(); + vi.unstubAllGlobals(); + } +}); + +it.each(["upload", "cancel", "failure", "retired", "blur", "unmount", "refocus"] as const)( + "qualified Android picker preserves the mounted viewer during background handoff (%s)", + async (outcome) => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + observed.mounts = 0; + observed.unmounts = 0; + observed.appState = "active"; + observed.focused = true; + observed.alert.mockClear(); + observed.renew.mockReset(); + observed.renew.mockResolvedValue({ query: { viewerTicket: "current-same-viewer" } }); + const chosen = Promise.withResolvers<{ + canceled: boolean; + assets: Array<{ uri: string; name: string; mimeType: string }>; + }>(); + const pickerOpened = Promise.withResolvers(); + observed.picker.mockImplementation(() => { + pickerOpened.resolve(); + return chosen.promise; + }); + const finished = Promise.withResolvers(); + const oldPost = Promise.withResolvers(); + const replacementPost = Promise.withResolvers(); + const replacementPosted = Promise.withResolvers(); + observed.alert.mockImplementation(() => finished.resolve()); + const posts: string[] = []; + vi.stubGlobal("fetch", async (url: string) => { + posts.push(url); + finished.resolve(); + if (outcome === "refocus") { + if (posts.length === 1) await oldPost.promise; + else { + replacementPosted.resolve(); + await replacementPost.promise; + } + } + return new Response(null, { status: 204 }); + }); + vi.stubGlobal( + "FormData", + class { + append() {} + }, + ); + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + const route = { + params: { + environmentId: "A", + threadId: "b", + resource: { + type: "preview" as const, + environmentId: EnvironmentId.make("B"), + threadId: ThreadId.make("b"), + tabId: "tab", + serverEpoch: "epoch", + }, + }, + }; + const state = (value: string) => { + observed.appState = value; + for (const listener of observed.listeners) listener(value); + }; + try { + await act(async () => root.render()); + expect(observed.mounts).toBe(1); + await act(async () => state("background")); + expect(observed.unmounts).toBe(1); + await act(async () => state("active")); + expect(observed.mounts).toBe(2); + await act(async () => + observed.message!({ + nativeEvent: { + data: JSON.stringify({ + type: "control", + controller: "you", + canOperate: true, + generation: 3, + dialog: null, + }), + }, + } as WebViewMessageEvent), + ); + await act(async () => + observed.message!({ + nativeEvent: { + data: JSON.stringify({ + type: "fileChooser", + chooser: { + multiple: false, + accept: "", + credentials: false, + uploadUrl: + "https://a.test/api/preview-stream/upload?threadId=b&tabId=tab&chooser=original&viewerTicket=expired", + }, + }), + }, + } as WebViewMessageEvent), + ); + const button = [...host.querySelectorAll("button")].find( + (value) => value.textContent === "Choose file", + )!; + await act(async () => { + button.click(); + await pickerOpened.promise; + }); + expect(isForegroundHandoffActive()).toBe(true); + await act(async () => state("background")); + expect( + observed.unmounts, + "Android picker background must not tear down its exact active viewer", + ).toBe(1); + expect(observed.renew).not.toHaveBeenCalled(); + if (outcome === "blur") { + await act(async () => { + observed.focused = false; + root.render(); + }); + expect(observed.unmounts).toBe(2); + } else if (outcome === "unmount") { + await act(async () => root.unmount()); + expect(observed.unmounts).toBe(2); + } + if (outcome === "refocus") { + await act(async () => { + state("active"); + chosen.resolve({ + canceled: false, + assets: [{ uri: "file:///chosen.txt", name: "chosen.txt", mimeType: "text/plain" }], + }); + await finished.promise; + }); + expect(isForegroundHandoffActive()).toBe(false); + await act(async () => state("background")); + expect(observed.unmounts, "the original viewer retains its own pending POST").toBe(1); + await act(async () => { + state("active"); + observed.focused = false; + root.render(); + }); + expect(observed.unmounts).toBe(2); + await act(async () => { + observed.focused = true; + root.render(); + }); + expect(observed.mounts).toBe(3); + await act(async () => state("background")); + expect( + observed.unmounts, + "a replacement must not inherit the retired viewer's pending POST exemption", + ).toBe(3); + await act(async () => state("active")); + expect(observed.mounts).toBe(4); + await act(async () => { + observed.message!({ + nativeEvent: { + data: JSON.stringify({ + type: "control", + controller: "you", + canOperate: true, + generation: 4, + dialog: null, + }), + }, + } as WebViewMessageEvent); + observed.message!({ + nativeEvent: { + data: JSON.stringify({ + type: "fileChooser", + chooser: { + multiple: false, + accept: "", + credentials: false, + uploadUrl: + "https://a.test/api/preview-stream/upload?threadId=b&tabId=tab&chooser=replacement&viewerTicket=expired", + }, + }), + }, + } as WebViewMessageEvent); + }); + await act(async () => { + [...host.querySelectorAll("button")] + .find((value) => value.textContent === "Choose file")! + .click(); + await replacementPosted.promise; + }); + expect(isForegroundHandoffActive()).toBe(false); + await act(async () => state("background")); + expect(observed.unmounts).toBe(3); + await act(async () => oldPost.resolve()); + expect( + observed.unmounts, + "retired promise settlement cannot end a replacement's own pending POST exemption", + ).toBe(3); + await act(async () => replacementPost.resolve()); + expect(observed.unmounts).toBe(4); + expect(posts.map((url) => new URL(url).searchParams.get("chooser"))).toEqual([ + "original", + "replacement", + ]); + expect(observed.renew).toHaveBeenCalledTimes(2); + return; + } + const refused = outcome === "retired" || outcome === "blur" || outcome === "unmount"; + if (refused) + observed.renew.mockRejectedValue(new Error("Original viewer no longer authorized")); + await act(async () => { + if (outcome === "failure") chosen.reject(new Error("Picker failed")); + else + chosen.resolve({ + canceled: outcome === "cancel", + assets: + outcome === "cancel" + ? [] + : [ + { + uri: "file:///chosen.txt", + name: "chosen.txt", + mimeType: "text/plain", + }, + ], + }); + await finished.promise; + }); + expect(isForegroundHandoffActive()).toBe(false); + expect(observed.unmounts, "true background must pause again after the handoff settles").toBe( + 2, + ); + expect(observed.mounts).toBe(2); + if (refused) { + expect(posts).toEqual([]); + expect(observed.renew).toHaveBeenCalledTimes(1); + expect(observed.alert).toHaveBeenLastCalledWith( + "Could not send the files to the page", + "Original viewer no longer authorized", + ); + } else if (outcome === "failure") { + expect(posts).toEqual([]); + expect(observed.renew).not.toHaveBeenCalled(); + expect(observed.alert).toHaveBeenLastCalledWith( + "Could not send the files to the page", + "Picker failed", + ); + } else { + expect(posts).toHaveLength(1); + expect(observed.renew).toHaveBeenCalledTimes(1); + const url = new URL(posts[0]!); + expect(url.searchParams.get("chooser")).toBe("original"); + expect(url.searchParams.get("viewerTicket")).toBe("current-same-viewer"); + } + } finally { + await act(async () => { + chosen.resolve({ canceled: true, assets: [] }); + oldPost.resolve(); + replacementPost.resolve(); + }); + await act(async () => root.unmount()); + host.remove(); + observed.alert.mockReset(); + vi.unstubAllGlobals(); + } + }, +); diff --git a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx index bff1ce2e58..93e52d067e 100644 --- a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx +++ b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx @@ -3,14 +3,20 @@ import * as Dialog from "../../components/ui/Dialog"; import previewStreamScript from "@t3tools/mobile-preview-stream"; import { previewStreamControlLabel, + resolvePreviewStreamFileUrl, previewStreamHostSetupMessage, type PreviewStreamControl, type PreviewStreamDownload, type PreviewStreamFileChooser, type PreviewStreamInput, } from "@t3tools/client-runtime/preview/server-browser-stream"; -import type { EnvironmentId, PreviewStreamHostSetup } from "@t3tools/contracts"; +import type { + EnvironmentId, + PreviewStreamHostSetup, + PreviewViewerResource, +} from "@t3tools/contracts"; import { + useCallback, useEffect, useEffectEvent, useImperativeHandle, @@ -50,12 +56,14 @@ type NativeStreamBridge = { readonly ref?: Ref; readonly onUnauthorized: () => void; readonly onGone?: () => void; + readonly onPopup?: (resource: PreviewViewerResource) => void; readonly onViewport?: (viewport: { readonly width: number; readonly height: number }) => void; readonly onControl?: (control: PreviewStreamControl | null) => void; readonly onPictureInPicture?: (state: PreviewPictureInPictureState, detail?: string) => void; /** True while frames show, so commands reach the page. Pass a stable function. */ readonly onStreamingChange?: (streaming: boolean) => void; readonly compact?: boolean; + readonly onFileTransferChange?: (active: boolean) => void; }; async function sendFilesToPage(chooser: PreviewStreamFileChooser, pick: boolean) { @@ -78,7 +86,9 @@ async function sendFilesToPage(chooser: PreviewStreamFileChooser, pick: boolean) } as unknown as Blob); } } - const response = await fetch(chooser.uploadUrl, { + const url = + chooser.resolveUploadUrl === undefined ? chooser.uploadUrl : await chooser.resolveUploadUrl(); + const response = await fetch(url, { method: "POST", body, credentials: chooser.credentials ? "include" : "omit", @@ -92,11 +102,12 @@ function offerDownload(download: PreviewStreamDownload) { { text: "Save or share", onPress: () => - void downloadAndShareAttachment({ - url: download.url, - attachment: { name: download.fileName, mimeType: "application/octet-stream" }, - signal: new AbortController().signal, - }).catch((cause: unknown) => + void (async () => + downloadAndShareAttachment({ + url: download.resolveUrl === undefined ? download.url : await download.resolveUrl(), + attachment: { name: download.fileName, mimeType: "application/octet-stream" }, + signal: new AbortController().signal, + }))().catch((cause: unknown) => Dialog.alert( "Could not save the download", cause instanceof Error ? cause.message : undefined, @@ -110,14 +121,33 @@ const MAX_REFUSALS = 3; export function PreviewStreamWebView( props: Omit & - Omit & { + Omit & { readonly environmentId: EnvironmentId; readonly paused?: boolean; + readonly backgrounded?: boolean; }, ) { - const { access, error, refresh } = usePreviewStreamAccess(props.environmentId); - if (access && !props.paused) { - return ; + const [pendingFiles, setPendingFiles] = useState(0); + const fileTransferChanged = useCallback((active: boolean) => { + setPendingFiles((current) => current + (active ? 1 : -1)); + }, []); + // Android backgrounds the app for its document picker. Keep this exact viewer + // through current admission and its answer, but always honor focus loss. + const live = !props.paused && (!props.backgrounded || pendingFiles > 0); + const { access, error, refresh } = usePreviewStreamAccess( + props.environmentId, + props.resource, + live, + ); + if (access && live) { + return ( + + ); } return ( { const refusals = ++unauthorized.current; if (refusals >= MAX_REFUSALS) return false; @@ -220,20 +252,28 @@ function AuthorizedPreviewStream({ function PreviewStreamDocumentView({ ref, + access, configuration, background, compact, + interactive, onUnauthorized, onGone, + onPopup, + resource, onViewport, onControl, onPictureInPicture, onStreamingChange, + onFileTransferChange, onRetry, onStreaming, onRecoverProcess, }: Omit & { + readonly access: PreviewStreamConfiguration["access"]; readonly configuration: string; + readonly resource?: PreviewViewerResource; + readonly interactive: boolean; readonly background: string; readonly onUnauthorized: () => boolean; readonly onRetry: () => void; @@ -252,16 +292,24 @@ function PreviewStreamDocumentView({ const dialogRef = useRef(null); const [promptText, setPromptText] = useState(""); const [fileChooser, setFileChooser] = useState(null); + const pendingAnswers = useRef(new Set<() => void>()); const answerFileChooser = (pick: boolean) => { const chooser = fileChooser; if (!chooser) return; setFileChooser(null); - void sendFilesToPage(chooser, pick).catch((cause: unknown) => - Dialog.alert( - "Could not send the files to the page", - cause instanceof Error ? cause.message : undefined, - ), - ); + const finish = () => { + if (pendingAnswers.current.delete(finish)) onFileTransferChange?.(false); + }; + pendingAnswers.current.add(finish); + onFileTransferChange?.(true); + void sendFilesToPage(chooser, pick) + .catch((cause: unknown) => + Dialog.alert( + "Could not send the files to the page", + cause instanceof Error ? cause.message : undefined, + ), + ) + .finally(finish); }; const controlChanged = useEffectEvent((next: PreviewStreamControl | null) => onControl?.(next)); const command = (input: PreviewStreamInput) => @@ -304,6 +352,9 @@ function PreviewStreamDocumentView({ const view = webView.current; return () => { active.current = false; + // A pending answer can retain only this opening. Its later settlement + // must not change the lifetime of a replacement stream. + for (const finish of pendingAnswers.current) finish(); view?.injectJavaScript("window.T3PreviewStream?.stop(); true;"); }; }, []); @@ -320,7 +371,7 @@ function PreviewStreamDocumentView({ {previewStreamControlLabel(control)} - {control?.canOperate ? ( + {interactive && control?.canOperate ? ( undefined); return; case "download": - offerDownload(message); + offerDownload({ + ...message, + resolveUrl: () => resolvePreviewStreamFileUrl(message.url, access), + }); return; case "fileChooser": - setFileChooser(message.chooser); + setFileChooser( + message.chooser === null + ? null + : { + ...message.chooser, + resolveUploadUrl: () => + resolvePreviewStreamFileUrl(message.chooser!.uploadUrl, access), + }, + ); return; case "pictureInPicture": onPictureInPicture?.(message, message.detail); diff --git a/apps/mobile/src/features/browser/preview-stream-document.test.ts b/apps/mobile/src/features/browser/preview-stream-document.test.ts new file mode 100644 index 0000000000..fa63e25b34 --- /dev/null +++ b/apps/mobile/src/features/browser/preview-stream-document.test.ts @@ -0,0 +1,61 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; +import { previewStreamDocument, previewStreamMessage } from "./preview-stream-document"; + +describe("native browser resource messages", () => { + const resource = { + type: "preview", + environmentId: EnvironmentId.make("destination-b"), + threadId: ThreadId.make("b-thread"), + tabId: "popup", + serverEpoch: "epoch-b", + }; + it("retains the qualified popup and refuses malformed or differently named resources", () => { + expect( + previewStreamMessage(JSON.stringify({ type: "popup", tabId: "popup", resource })), + ).toEqual({ type: "popup", tabId: "popup", resource }); + expect( + previewStreamMessage( + JSON.stringify({ + type: "popup", + tabId: "popup", + resource: { ...resource, tabId: "rival" }, + }), + ), + ).toBeNull(); + expect( + previewStreamMessage( + JSON.stringify({ + type: "popup", + tabId: "popup", + resource: { ...resource, ticket: "private" }, + }), + ), + ).toBeNull(); + expect( + previewStreamMessage( + JSON.stringify({ + type: "popup", + tabId: "popup", + resource: { ...resource, serverEpoch: "" }, + }), + ), + ).toBeNull(); + expect(previewStreamMessage(JSON.stringify({ type: "popup", tabId: "" }))).toBeNull(); + expect(previewStreamMessage('{"type":"popup","tabId":"legacy-popup"}')).toEqual({ + type: "popup", + tabId: "legacy-popup", + }); + expect(previewStreamMessage('{"type":"gone"}')).toEqual({ type: "gone" }); + }); + it("embeds resource/session values as data without introducing a new script", () => { + const configuration = JSON.stringify({ + resource, + access: { query: { viewerTicket: "" } }, + }); + const document = previewStreamDocument(configuration, "var T3PreviewStream={start(){}};"); + expect(document.match(/