From ec0481ed96999d873ff54b1e7221346195cbad77 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 11 Sep 2026 12:31:14 -0700 Subject: [PATCH 1/4] feat(web): float device streams over chat The floating preview mirrored only browser tabs, so watching an agent drive a simulator meant keeping the right panel open on the device tab. The mini player now takes a source: a browser tab or a device stream. Agent-opened devices float over chat like agent-driven browsers (a right-panel tab when auto-show is off), the device panel toolbar can float the active device, and closing the panel on a device floats it instead of dropping it. The floating device follows the stream's reported screen size, rotates with the device, and closes when its session ends. Co-Authored-By: Claude Code --- .../web/src/components/ChatView.logic.test.ts | 49 ++- apps/web/src/components/ChatView.logic.ts | 19 +- apps/web/src/components/ChatView.tsx | 79 +++-- .../web/src/components/device/DevicePanel.tsx | 18 ++ .../preview/PreviewAutomationHosts.tsx | 18 +- .../components/preview/PreviewView.test.tsx | 14 +- .../src/components/preview/PreviewView.tsx | 18 +- .../preview/ThreadPreviewMiniPlayer.tsx | 286 +++++++++++++----- .../preview/previewMiniPlayerLayout.test.ts | 32 ++ .../preview/previewMiniPlayerLayout.ts | 25 +- .../settings/IntegrationsSettings.tsx | 2 +- .../src/components/settings/settingsSearch.ts | 2 +- apps/web/src/previewMiniPlayerStore.test.ts | 68 +++-- apps/web/src/previewMiniPlayerStore.ts | 73 ++++- docs/user/devices.md | 10 +- 15 files changed, 550 insertions(+), 163 deletions(-) diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index 973dd5749027..ae0cdc8581ff 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -126,7 +126,7 @@ describe("floating browser preview", () => { const ref = scopeThreadRef(EnvironmentId.make("env-1"), ThreadId.make("thread-1")); const panels = useRightPanelStore.getState(); const revision = panels.getUserActionRevision(ref); - usePreviewMiniPlayerStore.getState().open(ref, "agent-tab"); + usePreviewMiniPlayerStore.getState().open(ref, { kind: "browser", tabId: "agent-tab" }); panels.reconcileBrowserSurfaces(ref, ["agent-tab"]); const intent = selectThreadPreviewMiniPlayer( usePreviewMiniPlayerStore.getState().byThreadKey, @@ -135,7 +135,7 @@ describe("floating browser preview", () => { const isFloating = () => shouldRenderPreviewMiniPlayer( selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, ref) - ?.tabId ?? null, + ?.source ?? null, selectActiveRightPanelSurface(useRightPanelStore.getState().byThreadKey, ref), ); @@ -152,22 +152,61 @@ describe("floating browser preview", () => { }); it("only hides the duplicate while the same browser is rendered in the panel", () => { + const tab = { kind: "browser", tabId: "tab-1" } as const; expect(shouldRenderPreviewMiniPlayer(null, null)).toBe(false); expect( - shouldRenderPreviewMiniPlayer("tab-1", { + shouldRenderPreviewMiniPlayer(tab, { id: "browser:one", kind: "preview", resourceId: "tab-1", }), ).toBe(false); expect( - shouldRenderPreviewMiniPlayer("tab-1", { + shouldRenderPreviewMiniPlayer(tab, { id: "browser:two", kind: "preview", resourceId: "tab-2", }), ).toBe(true); - expect(shouldRenderPreviewMiniPlayer("tab-1", { id: "diff", kind: "diff" })).toBe(true); + expect(shouldRenderPreviewMiniPlayer(tab, { id: "diff", kind: "diff" })).toBe(true); + }); + + it("only hides a floating device while that device is rendered in the panel", () => { + const pixel = { + kind: "device", + hostId: "nucbox", + deviceId: "emulator-5580", + platform: "android", + name: "Pixel", + } as const; + const target = { + hostId: "nucbox", + deviceId: "emulator-5580", + platform: "android", + name: "Pixel", + } as const; + expect( + shouldRenderPreviewMiniPlayer(pixel, { + id: "device:nucbox:emulator-5580", + kind: "device", + target, + }), + ).toBe(false); + expect( + shouldRenderPreviewMiniPlayer(pixel, { + id: "device:nucbox:emulator-5554", + kind: "device", + target: { ...target, deviceId: "emulator-5554" }, + }), + ).toBe(true); + expect(shouldRenderPreviewMiniPlayer(pixel, { id: "device", kind: "device" })).toBe(true); + expect( + shouldRenderPreviewMiniPlayer(pixel, { + id: "browser:one", + kind: "preview", + resourceId: "emulator-5580", + }), + ).toBe(true); }); }); diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index 772a0f3cf2fa..b6bb98b4c50a 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -50,6 +50,7 @@ import { import type { DraftThreadEnvMode } from "../composerDraftStore"; import type { ComposerSubmissionIntent } from "../composer-logic"; import type { TimelineEntry } from "../session-logic"; +import type { PreviewMiniPlayerSource } from "../previewMiniPlayerStore"; import type { DesktopPreviewOverlay } from "../previewStateStore"; import type { RightPanelSurface } from "../rightPanelStore"; import { @@ -88,16 +89,22 @@ export function agentControlledBrowserCloseConfirmation( ].join("\n"); } +/** The floating player hides only while the same source is rendered in the panel. */ export function shouldRenderPreviewMiniPlayer( - miniPlayerTabId: string | null, + source: PreviewMiniPlayerSource | null, renderedRightPanelSurface: RightPanelSurface | null, ): boolean { - return ( - miniPlayerTabId !== null && - !( + if (source === null) return false; + if (source.kind === "browser") { + return !( renderedRightPanelSurface?.kind === "preview" && - renderedRightPanelSurface.resourceId === miniPlayerTabId - ) + renderedRightPanelSurface.resourceId === source.tabId + ); + } + return !( + renderedRightPanelSurface?.kind === "device" && + renderedRightPanelSurface.target?.hostId === source.hostId && + renderedRightPanelSurface.target.deviceId === source.deviceId ); } diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index fc529b02a73b..bae0d1eb98e1 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -195,6 +195,8 @@ import { useSidebarPendingFileDropStore, } from "../sidebarPendingFileDropStore"; import { + browserMiniPlayerSource, + previewMiniPlayerSourceKey, selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore, } from "../previewMiniPlayerStore"; @@ -566,6 +568,8 @@ const PreviewPanel = lazy(() => import("./preview/PreviewPanel").then((module) => ({ default: module.PreviewPanel })), ); const DiffPanel = lazy(() => import("./DiffPanel")); +const selectAutoShowFloatingPreview = (settings: { browserAutoShowFloatingPreview: boolean }) => + settings.browserAutoShowFloatingPreview; const DevicePanel = lazy(() => import("./device/DevicePanel").then((module) => ({ default: module.DevicePanel })), ); @@ -1958,7 +1962,7 @@ export default function ChatView(props: ChatViewProps) { const renderedRightPanelSurface = rightPanelPresence.value?.activeSurface ?? null; const renderedRightPanelSurfaces = rightPanelPresence.value?.surfaces ?? []; const previewMiniPlayerVisible = shouldRenderPreviewMiniPlayer( - activePreviewMiniPlayer?.tabId ?? null, + activePreviewMiniPlayer?.source ?? null, renderedRightPanelSurface, ); const canMaximizeRightPanel = rightPanelOpen && !shouldUseRightPanelSheet; @@ -1974,8 +1978,10 @@ export default function ChatView(props: ChatViewProps) { }, [activePreviewState.sessions, activeThreadRef]); useEffect(() => { - if (!activeThreadRef || !activePreviewMiniPlayer) return; - const miniTabStillExists = Boolean(activePreviewState.sessions[activePreviewMiniPlayer.tabId]); + if (!activeThreadRef || activePreviewMiniPlayer?.source.kind !== "browser") return; + const miniTabStillExists = Boolean( + activePreviewState.sessions[activePreviewMiniPlayer.source.tabId], + ); if (!miniTabStillExists) { usePreviewMiniPlayerStore.getState().close(activeThreadRef); } @@ -4215,9 +4221,12 @@ export default function ChatView(props: ChatViewProps) { } useRightPanelStore.getState().open(activeThreadRef, "device"); }, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); - // Reconcile new server sessions into separate tabs, including sessions opened - // by an agent or another client. The first snapshot is a baseline: persisted - // tabs restore themselves, and existing sessions must not resurrect closed tabs. + // A device the agent opens floats over chat like an agent-driven browser, + // or becomes a panel tab when floating previews are off. Sessions opened by + // another client arrive the same way. The first snapshot is a baseline: + // persisted tabs restore themselves, and existing sessions must not + // resurrect closed tabs. + const autoShowFloatingPreview = useClientSettings(selectAutoShowFloatingPreview); const previousDeviceSessions = useRef(new Map>()); useEffect(() => { if (!activeThreadRef || !deviceStateLoaded) return; @@ -4228,9 +4237,24 @@ export default function ChatView(props: ChatViewProps) { const key = (session: (typeof sessions)[number]) => `${session.hostId}:${session.deviceId}`; const previous = previousDeviceSessions.current.get(threadKey); previousDeviceSessions.current.set(threadKey, new Set(sessions.map(key))); - if (!previous || shouldUseRightPanelSheet) return; + if (!previous) return; for (const session of sessions) { if (previous?.has(key(session))) continue; + const device = deviceState.devices.find( + (entry) => entry.hostId === session.hostId && entry.id === session.deviceId, + ); + if (!device) continue; + const target = { + hostId: session.hostId, + deviceId: session.deviceId, + platform: device.platform, + name: device.name, + }; + if (autoShowFloatingPreview) { + usePreviewMiniPlayerStore.getState().open(activeThreadRef, { kind: "device", ...target }); + continue; + } + if (shouldUseRightPanelSheet) continue; const existing = useRightPanelStore .getState() .byThreadKey[scopedThreadKey(activeThreadRef)]?.surfaces.some( @@ -4240,28 +4264,30 @@ export default function ChatView(props: ChatViewProps) { surface.target.deviceId === session.deviceId, ); if (existing) continue; - const device = deviceState.devices.find( - (entry) => entry.hostId === session.hostId && entry.id === session.deviceId, - ); - if (!device) continue; - useRightPanelStore.getState().openDevice( - activeThreadRef, - { - hostId: session.hostId, - deviceId: session.deviceId, - platform: device.platform, - name: device.name, - }, - true, - ); + useRightPanelStore.getState().openDevice(activeThreadRef, target, true); } }, [ activeThreadRef, + autoShowFloatingPreview, deviceStateLoaded, shouldUseRightPanelSheet, deviceState.sessions, deviceState.devices, ]); + // A floating device follows its session: once the agent or another client + // closes the device there is nothing left to stream. + useEffect(() => { + if (!activeThreadRef || !deviceStateLoaded) return; + const source = activePreviewMiniPlayer?.source; + if (source?.kind !== "device") return; + const sessionStillExists = deviceState.sessions.some( + (session) => + session.threadId === activeThreadRef.threadId && + session.hostId === source.hostId && + session.deviceId === source.deviceId, + ); + if (!sessionStillExists) usePreviewMiniPlayerStore.getState().close(activeThreadRef); + }, [activePreviewMiniPlayer, activeThreadRef, deviceState.sessions, deviceStateLoaded]); const openFileSurface = useCallback( (relativePath: string) => { if (!activeThreadRef || !activeProject) return; @@ -4421,10 +4447,15 @@ export default function ChatView(props: ChatViewProps) { ]); const closePreviewPanel = useCallback(() => { if (activeThreadRef) { + // Closing the panel on a live browser or device floats it instead of dropping it. if (activeRightPanelSurface?.kind === "preview" && activeRightPanelSurface.resourceId) { usePreviewMiniPlayerStore .getState() - .open(activeThreadRef, activeRightPanelSurface.resourceId); + .open(activeThreadRef, browserMiniPlayerSource(activeRightPanelSurface.resourceId)); + } else if (activeRightPanelSurface?.kind === "device" && activeRightPanelSurface.target) { + usePreviewMiniPlayerStore + .getState() + .open(activeThreadRef, { kind: "device", ...activeRightPanelSurface.target }); } setMaximizedRightPanelThreadKey(null); useRightPanelStore.getState().close(activeThreadRef); @@ -8834,9 +8865,9 @@ export default function ChatView(props: ChatViewProps) { {activeThreadRef && activePreviewMiniPlayer && previewMiniPlayerVisible ? ( ) : null} diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index 5b312f7fc4f4..950a202a7b9f 100644 --- a/apps/web/src/components/device/DevicePanel.tsx +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -7,6 +7,7 @@ import type { import { ChevronLeft, Home, + PictureInPicture2, Power, RotateCcw, SlidersHorizontal, @@ -16,6 +17,7 @@ import { } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; +import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; import { useRightPanelStore, type RightPanelSurface } from "~/rightPanelStore"; import { Button } from "~/components/ui/button"; import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list"; @@ -116,6 +118,19 @@ export function DevicePanel(props: { } }; + // Floating the device closes the panel, like the browser's floating preview. + const floatActive = () => { + if (!activeDevice) return; + usePreviewMiniPlayerStore.getState().open(props.threadRef, { + kind: "device", + hostId: activeDevice.hostId, + deviceId: activeDevice.id, + platform: activeDevice.platform, + name: activeDevice.name, + }); + useRightPanelStore.getState().close(props.threadRef); + }; + const closeActive = (powerOff: boolean) => { if (!powerOff) { useRightPanelStore.getState().closeSurface(props.threadRef, props.surface.id); @@ -218,6 +233,9 @@ export function DevicePanel(props: { > + + + closeActive(true)}> diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index d1fc12821730..a793e8a2c8c0 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -29,7 +29,11 @@ import { reconcilePreviewServerSessions, updatePreviewServerSnapshot, } from "~/previewStateStore"; -import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { + browserMiniPlayerSource, + selectThreadPreviewMiniPlayerTabId, + usePreviewMiniPlayerStore, +} from "~/previewMiniPlayerStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; import { readActiveBrowserRecordingTargets, @@ -378,7 +382,9 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) ?.has(runtimeTabId) ?? false, }) ) { - usePreviewMiniPlayerStore.getState().open(threadRef, readyTabId); + usePreviewMiniPlayerStore + .getState() + .open(threadRef, browserMiniPlayerSource(readyTabId)); } } browserActivity.release ??= acquireBrowserSurfaceActivity(runtimeTabId); @@ -493,11 +499,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) new Set([activeRuntimeTabId]), ); } - const miniPlayer = selectThreadPreviewMiniPlayer( + const miniPlayerTabId = selectThreadPreviewMiniPlayerTabId( usePreviewMiniPlayerStore.getState().byThreadKey, threadRef, ); - if (miniPlayer?.tabId === activeTabId) { + if (miniPlayerTabId === activeTabId) { usePreviewMiniPlayerStore.getState().close(threadRef); } } else if (shouldPresentPreview) { @@ -507,7 +513,9 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } } if (shouldPresentPreview) { - usePreviewMiniPlayerStore.getState().open(threadRef, activeTabId); + usePreviewMiniPlayerStore + .getState() + .open(threadRef, browserMiniPlayerSource(activeTabId)); } if (activeSnapshot && previewAutomationOpenNeedsOverlay(input, activeSnapshot)) { await requireReadyTab(); diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 2a146834012e..ea28a93235ee 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -171,7 +171,7 @@ vi.mock("~/previewMiniPlayerStore", () => { byThreadKey: mocks.miniPlayerTabId ? { "environment-1:thread-1": { - tabId: mocks.miniPlayerTabId, + source: { kind: "browser", tabId: mocks.miniPlayerTabId }, position: null, }, } @@ -185,9 +185,10 @@ vi.mock("~/previewMiniPlayerStore", () => { }, ); return { - selectThreadPreviewMiniPlayer: ( - byThreadKey: Record, - ) => byThreadKey["environment-1:thread-1"] ?? null, + browserMiniPlayerSource: (tabId: string) => ({ kind: "browser", tabId }), + selectThreadPreviewMiniPlayerTabId: ( + byThreadKey: Record, + ) => byThreadKey["environment-1:thread-1"]?.source.tabId ?? null, usePreviewMiniPlayerStore, }; }); @@ -485,7 +486,10 @@ describe("PreviewView navigation", () => { renderToStaticMarkup(); expect(mocks.pictureInPicturePressed).toBe(false); mocks.togglePictureInPicture?.(); - expect(mocks.openMiniPlayer).toHaveBeenCalledWith(props.threadRef, "tab-1"); + expect(mocks.openMiniPlayer).toHaveBeenCalledWith(props.threadRef, { + kind: "browser", + tabId: "tab-1", + }); expect(mocks.closeRightPanel).toHaveBeenCalledWith(props.threadRef); mocks.miniPlayerTabId = "tab-1"; diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index e6ad2758bc48..6086e049ab11 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -34,7 +34,11 @@ import { resolveDiscoveredServerUrl } from "~/browser/browserTargetResolver"; import { useEnvironmentHttpBaseUrl } from "~/state/environments"; import { previewEnvironment } from "~/state/preview"; import { useAtomCommand } from "~/state/use-atom-command"; -import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { + browserMiniPlayerSource, + selectThreadPreviewMiniPlayerTabId, + usePreviewMiniPlayerStore, +} from "~/previewMiniPlayerStore"; import { useRightPanelStore } from "~/rightPanelStore"; import { previewBridge } from "./previewBridge"; @@ -114,8 +118,8 @@ export function PreviewView({ threadRef, BROWSER_HISTORY_MAX_ENTRIES_PER_PROJECT, ); - const miniPlayer = usePreviewMiniPlayerStore((state) => - selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), + const miniPlayerTabId = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayerTabId(state.byThreadKey, threadRef), ); const addPreviewAnnotation = useComposerDraftStore((store) => store.addPreviewAnnotation); const addImage = useComposerDraftStore((store) => store.addImage); @@ -311,13 +315,13 @@ export function PreviewView({ const handlePictureInPicture = useCallback(() => { if (!tabId) return; - if (miniPlayer?.tabId === tabId) { + if (miniPlayerTabId === tabId) { usePreviewMiniPlayerStore.getState().close(threadRef); return; } - usePreviewMiniPlayerStore.getState().open(threadRef, tabId); + usePreviewMiniPlayerStore.getState().open(threadRef, browserMiniPlayerSource(tabId)); useRightPanelStore.getState().close(threadRef); - }, [miniPlayer?.tabId, tabId, threadRef]); + }, [miniPlayerTabId, tabId, threadRef]); const handleNativePictureInPicture = useCallback(() => { if (!previewBridge || !runtimeTabId) return; @@ -722,7 +726,7 @@ export function PreviewView({ captureDisabled={!desktopOverlay || isUnreachable} recording={recordingRuntimeTabId !== null} onPictureInPicture={previewBridge && tabId ? handlePictureInPicture : undefined} - pictureInPicture={miniPlayer?.tabId === tabId} + pictureInPicture={miniPlayerTabId === tabId} pictureInPictureDisabled={!desktopOverlay?.hasWebContents || isUnreachable} onPickElement={previewBridge && tabId ? handlePickElement : undefined} pickActive={pickActive} diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index 4384019abbad..bbd219bca3cb 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -2,7 +2,13 @@ import { FILL_PREVIEW_VIEWPORT, type ScopedThreadRef } from "@t3tools/contracts"; import { PanelRightIcon, PictureInPicture2, XIcon } from "lucide-react"; -import { type PointerEvent as ReactPointerEvent, useLayoutEffect, useRef, useState } from "react"; +import { + type PointerEvent as ReactPointerEvent, + type ReactNode, + useLayoutEffect, + useRef, + useState, +} from "react"; import { BrowserSurfaceSlot } from "~/browser/BrowserSurfaceSlot"; import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; @@ -15,17 +21,23 @@ import { cn } from "~/lib/utils"; import { useThreadPreviewState } from "~/previewStateStore"; import { type PreviewMiniPlayerSize, - selectThreadPreviewMiniPlayer, + type PreviewMiniPlayerSource, + type PreviewMiniPlayerState, + previewMiniPlayerSourceKey, usePreviewMiniPlayerStore, } from "~/previewMiniPlayerStore"; import { useRightPanelStore } from "~/rightPanelStore"; +import { useDeviceState } from "~/state/device"; +import { DeviceStreamView } from "../device/DeviceStreamView"; +import type { DeviceScreenSize } from "../device/deviceStream"; import { previewBridge } from "./previewBridge"; import { clampPreviewMiniPlayerPosition, PREVIEW_MINI_PLAYER_WEBVIEW_Z_INDEX, type PreviewMiniPlayerFrame, resizePreviewMiniPlayer, + resolveDeviceMiniPlayerSourceSize, resolvePreviewMiniPlayerFrame, resolvePreviewMiniPlayerSourceSize, } from "./previewMiniPlayerLayout"; @@ -40,7 +52,7 @@ interface PointerGesture { interface Props { readonly threadRef: ScopedThreadRef; - readonly tabId: string; + readonly miniPlayer: PreviewMiniPlayerState; readonly bottomInset: number; } @@ -61,17 +73,34 @@ const RESIZE_HANDLES: ReadonlyArray<{ { direction: "southeast", className: "-bottom-2 -right-2 size-4 cursor-nwse-resize" }, ]; -/** - * Floats the thread's browser surface over chat. Native clipping and the DOM - * frame use the same radius so their separately composited edges stay aligned. - */ -export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props) { - const containerRef = useRef(null); - const gestureRef = useRef(null); - const [container, setContainer] = useState(null); - const miniPlayer = usePreviewMiniPlayerStore((state) => - selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), +/** Floats the thread's browser tab or device stream over chat. */ +export function ThreadPreviewMiniPlayer({ threadRef, miniPlayer, bottomInset }: Props) { + const { source } = miniPlayer; + return source.kind === "browser" ? ( + + ) : ( + ); +} + +function BrowserMiniPlayer({ + threadRef, + tabId, + miniPlayer, + bottomInset, +}: Props & { readonly tabId: string }) { const previewState = useThreadPreviewState(threadRef); const snapshot = previewState.sessions[tabId] ?? null; const runtimeTabId = previewRuntimeTabId(threadRef, previewState.serverEpoch, tabId); @@ -79,25 +108,11 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props const fittedSourceContent = useBrowserSurfaceStore( (state) => state.byTabId[runtimeTabId]?.fittedSourceContent ?? null, ); - const source = resolvePreviewMiniPlayerSourceSize( + const sourceSize = resolvePreviewMiniPlayerSourceSize( snapshot?.viewport ?? FILL_PREVIEW_VIEWPORT, fittedSourceContent, desktopOverlay?.zoomFactor ?? 1, ); - const frame = - container && miniPlayer?.tabId === tabId - ? resolvePreviewMiniPlayerFrame({ - width: miniPlayer.width, - position: miniPlayer.position, - source, - container, - bottomInset, - }) - : null; - - const close = () => { - usePreviewMiniPlayerStore.getState().close(threadRef); - }; const openInPanel = () => { usePreviewMiniPlayerStore.getState().close(threadRef); @@ -118,6 +133,164 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props }); }; + if (!snapshot) return null; + + return ( + + event.stopPropagation()} + onClick={toggleNativePictureInPicture} + /> + } + > + + + + {desktopOverlay?.pictureInPicture + ? "Close separate window" + : "Pop into separate window"} + + + } + > + {(frame) => ( + <> + + {!desktopOverlay?.hasWebContents ? ( +
+ Reconnecting preview… +
+ ) : null} + + )} + + ); +} + +function DeviceMiniPlayer({ + threadRef, + source, + miniPlayer, + bottomInset, +}: Props & { readonly source: Extract }) { + const { state: deviceState } = useDeviceState(threadRef.environmentId); + const [screen, setScreen] = useState(null); + const sourceSize = resolveDeviceMiniPlayerSourceSize(source.platform, screen); + const device = deviceState.devices.find( + (entry) => entry.hostId === source.hostId && entry.id === source.deviceId, + ); + const hostLabel = + deviceState.hosts.find((host) => host.id === source.hostId)?.label ?? "Device host"; + + const openInPanel = () => { + usePreviewMiniPlayerStore.getState().close(threadRef); + useRightPanelStore.getState().openDevice(threadRef, { + hostId: source.hostId, + deviceId: source.deviceId, + platform: source.platform, + name: source.name, + }); + }; + + return ( + + {() => ( + // The stream is DOM, so it takes the band the browser's native webview would. +
+ +
+ )} + + ); +} + +/** + * The frame, drag/resize gestures, and hover pill shared by every floating + * source. Native clipping and the DOM frame use the same radius so their + * separately composited edges stay aligned. + */ +function MiniPlayerShell({ + threadRef, + miniPlayer, + sourceSize, + bottomInset, + label, + onOpenInPanel, + pillActions, + children, +}: { + readonly threadRef: ScopedThreadRef; + readonly miniPlayer: PreviewMiniPlayerState; + readonly sourceSize: PreviewMiniPlayerSize; + readonly bottomInset: number; + readonly label: string; + readonly onOpenInPanel: () => void; + readonly pillActions?: ReactNode; + readonly children: (frame: PreviewMiniPlayerFrame) => ReactNode; +}) { + const containerRef = useRef(null); + const gestureRef = useRef(null); + const [container, setContainer] = useState(null); + const sourceKey = previewMiniPlayerSourceKey(miniPlayer.source); + const frame = container + ? resolvePreviewMiniPlayerFrame({ + width: miniPlayer.width, + position: miniPlayer.position, + source: sourceSize, + container, + bottomInset, + }) + : null; + + const close = () => { + usePreviewMiniPlayerStore.getState().close(threadRef); + }; + useLayoutEffect(() => { const element = containerRef.current; if (!element) return; @@ -160,7 +333,7 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props if (gesture.direction === null) { store.move( threadRef, - tabId, + sourceKey, clampPreviewMiniPlayerPosition( { x: gesture.frame.x + delta.x, y: gesture.frame.y + delta.y }, container, @@ -174,12 +347,12 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props start: gesture.frame, direction: gesture.direction, delta, - source, + source: sourceSize, container, bottomInset, }); - store.resize(threadRef, tabId, next.width); - store.move(threadRef, tabId, { x: next.x, y: next.y }); + store.resize(threadRef, sourceKey, next.width); + store.move(threadRef, sourceKey, { x: next.x, y: next.y }); }; const endGesture = (event: ReactPointerEvent) => { @@ -190,14 +363,12 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props } }; - if (!snapshot || miniPlayer?.tabId !== tabId) return null; - return (
{frame ? (
event.stopPropagation()} - onClick={openInPanel} + onClick={onOpenInPanel} /> } > @@ -235,31 +406,7 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props Open in right panel - - event.stopPropagation()} - onClick={toggleNativePictureInPicture} - /> - } - > - - - - {desktopOverlay?.pictureInPicture - ? "Close separate window" - : "Pop into separate window"} - - + {pillActions}
- + {children(frame)}
- {!desktopOverlay?.hasWebContents ? ( -
- Reconnecting preview… -
- ) : null} {RESIZE_HANDLES.map(({ direction, className }) => (
{ }); }); +describe("resolveDeviceMiniPlayerSourceSize", () => { + it("stands in with the platform's phone shape until the stream reports a size", () => { + const ios = resolveDeviceMiniPlayerSourceSize("ios", null); + expect(ios.width / ios.height).toBeCloseTo(9 / 19.5); + const android = resolveDeviceMiniPlayerSourceSize("android", null); + expect(android.width / android.height).toBeCloseTo(9 / 20); + }); + + it("turns a rotated screen into a landscape box", () => { + const screen = { width: 1_179, height: 2_556, orientation: "landscape_left" } as const; + expect(resolveDeviceMiniPlayerSourceSize("ios", screen)).toEqual({ + width: 2_556, + height: 1_179, + }); + expect( + resolveDeviceMiniPlayerSourceSize("android", { ...screen, orientation: "portrait" }), + ).toEqual({ width: 1_179, height: 2_556 }); + }); + + it("floats a phone at the minimum width rather than the default box", () => { + expect( + resolvePreviewMiniPlayerFrame({ + width: null, + position: null, + source: resolveDeviceMiniPlayerSourceSize("ios", null), + container, + }), + ).toMatchObject({ width: 240, height: 520 }); + }); +}); + describe("resolvePreviewMiniPlayerFrame", () => { it("opens at the source aspect ratio in the top-right corner", () => { expect( diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index 372ed71c6b0d..108b8a9d846e 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -1,4 +1,4 @@ -import type { PreviewViewportSetting } from "@t3tools/contracts"; +import type { DevicePlatform, PreviewViewportSetting } from "@t3tools/contracts"; import type { BrowserSurfaceContentPresentation } from "~/browser/browserSurfaceStore"; import { @@ -7,6 +7,8 @@ import { } from "~/browser/browserViewportLayout"; import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; +import type { DeviceScreenSize } from "../device/deviceStream"; + export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; // The mini-player shell straddles this webview at 47 and 49; dialogs begin at 50. export const PREVIEW_MINI_PLAYER_WEBVIEW_Z_INDEX = 48; @@ -34,6 +36,27 @@ export function resolvePreviewMiniPlayerSourceSize( }; } +/** + * The device screen as the user sees it, so a rotated phone floats as a + * landscape box. Before the stream reports its size the platform's usual phone + * shape stands in, matching the stream view's own placeholder aspect; the + * nominal width only keeps the source cap above any sensible player width. + */ +export function resolveDeviceMiniPlayerSourceSize( + platform: DevicePlatform, + screen: DeviceScreenSize | null, +): PreviewMiniPlayerSize { + if (!screen) { + const width = 1_000; + return { width, height: width / (platform === "ios" ? 9 / 19.5 : 9 / 20) }; + } + const landscape = + screen.orientation === "landscape_left" || screen.orientation === "landscape_right"; + const long = Math.max(screen.width, screen.height); + const short = Math.min(screen.width, screen.height); + return landscape ? { width: long, height: short } : { width: short, height: long }; +} + const availableArea = ( container: PreviewMiniPlayerSize, bottomInset: number, diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index 188c3045fbcb..4197a828e2ff 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -718,7 +718,7 @@ function BrowserAutoShowFloatingPreviewSetting({ disabled }: { readonly disabled return ( { usePreviewMiniPlayerStore.setState({ byThreadKey: {} }); @@ -13,52 +28,71 @@ beforeEach(() => { describe("previewMiniPlayerStore", () => { it("keeps floating previews scoped to their thread", () => { - usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); - usePreviewMiniPlayerStore.getState().open(refB, "tab-b"); + usePreviewMiniPlayerStore.getState().open(refA, tabA); + usePreviewMiniPlayerStore.getState().open(refB, tabB); expect( selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), - ).toMatchObject({ tabId: "tab-a" }); + ).toMatchObject({ source: tabA }); expect( selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refB), - ).toMatchObject({ tabId: "tab-b" }); + ).toMatchObject({ source: tabB }); }); it("preserves position when switching the floating tab within one thread", () => { - usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); - usePreviewMiniPlayerStore.getState().move(refA, "tab-a", { x: 24, y: 48 }); - usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + usePreviewMiniPlayerStore.getState().open(refA, tabA); + usePreviewMiniPlayerStore.getState().move(refA, "browser:tab-a", { x: 24, y: 48 }); + usePreviewMiniPlayerStore.getState().open(refA, tabB); expect( selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), ).toEqual({ - tabId: "tab-b", + source: tabB, position: { x: 24, y: 48 }, width: null, }); }); it("ignores stale drag updates after the floating tab changes", () => { - usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); - usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); - usePreviewMiniPlayerStore.getState().move(refA, "tab-a", { x: 100, y: 100 }); + usePreviewMiniPlayerStore.getState().open(refA, tabA); + usePreviewMiniPlayerStore.getState().open(refA, tabB); + usePreviewMiniPlayerStore.getState().move(refA, "browser:tab-a", { x: 100, y: 100 }); expect( selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), ).toEqual({ - tabId: "tab-b", + source: tabB, position: null, width: null, }); }); it("preserves a thread-bound width while switching tabs", () => { - usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); - usePreviewMiniPlayerStore.getState().resize(refA, "tab-a", 480); - usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + usePreviewMiniPlayerStore.getState().open(refA, tabA); + usePreviewMiniPlayerStore.getState().resize(refA, "browser:tab-a", 480); + usePreviewMiniPlayerStore.getState().open(refA, tabB); expect( selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), - ).toMatchObject({ tabId: "tab-b", width: 480 }); + ).toMatchObject({ source: tabB, width: 480 }); + }); + + it("floats one source per thread, so a device replaces the browser tab", () => { + usePreviewMiniPlayerStore.getState().open(refA, tabA); + usePreviewMiniPlayerStore.getState().open(refA, pixel); + const floating = selectThreadPreviewMiniPlayer( + usePreviewMiniPlayerStore.getState().byThreadKey, + refA, + ); + + expect(floating).toMatchObject({ source: pixel }); + expect( + selectThreadPreviewMiniPlayerTabId(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toBeNull(); + // The same device under a new label is still the same floating source. + usePreviewMiniPlayerStore.getState().open(refA, { ...pixel, name: "Renamed" }); + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toBe(floating); }); }); diff --git a/apps/web/src/previewMiniPlayerStore.ts b/apps/web/src/previewMiniPlayerStore.ts index aed9376af156..b3e3c3b0505f 100644 --- a/apps/web/src/previewMiniPlayerStore.ts +++ b/apps/web/src/previewMiniPlayerStore.ts @@ -1,5 +1,5 @@ import { scopedThreadKey } from "@t3tools/client-runtime/environment"; -import type { ScopedThreadRef } from "@t3tools/contracts"; +import type { DevicePlatform, ScopedThreadRef } from "@t3tools/contracts"; import { create } from "zustand"; export interface PreviewMiniPlayerPosition { @@ -12,34 +12,66 @@ export interface PreviewMiniPlayerSize { readonly height: number; } +/** What the floating player mirrors: a browser tab or a device stream. */ +export type PreviewMiniPlayerSource = + | { readonly kind: "browser"; readonly tabId: string } + | { + readonly kind: "device"; + readonly hostId: string; + readonly deviceId: string; + readonly platform: DevicePlatform; + readonly name: string; + }; + export interface PreviewMiniPlayerState { - readonly tabId: string; + readonly source: PreviewMiniPlayerSource; readonly position: PreviewMiniPlayerPosition | null; - /** Height always follows the previewed viewport's aspect ratio. */ + /** Height always follows the mirrored source's aspect ratio. */ readonly width: number | null; } interface PreviewMiniPlayerStoreState { readonly byThreadKey: Record; - readonly open: (ref: ScopedThreadRef, tabId: string) => void; + readonly open: (ref: ScopedThreadRef, source: PreviewMiniPlayerSource) => void; readonly close: (ref: ScopedThreadRef) => void; - readonly move: (ref: ScopedThreadRef, tabId: string, position: PreviewMiniPlayerPosition) => void; - readonly resize: (ref: ScopedThreadRef, tabId: string, width: number) => void; + /** `sourceKey` guards against a drag that outlives the source it started on. */ + readonly move: ( + ref: ScopedThreadRef, + sourceKey: string, + position: PreviewMiniPlayerPosition, + ) => void; + readonly resize: (ref: ScopedThreadRef, sourceKey: string, width: number) => void; readonly removeThread: (ref: ScopedThreadRef) => void; } +export function previewMiniPlayerSourceKey(source: PreviewMiniPlayerSource): string { + return source.kind === "browser" + ? `browser:${source.tabId}` + : `device:${encodeURIComponent(source.hostId)}:${encodeURIComponent(source.deviceId)}`; +} + +export const browserMiniPlayerSource = (tabId: string): PreviewMiniPlayerSource => ({ + kind: "browser", + tabId, +}); + export const usePreviewMiniPlayerStore = create()((set) => ({ byThreadKey: {}, - open: (ref, tabId) => + open: (ref, source) => set((state) => { const threadKey = scopedThreadKey(ref); const current = state.byThreadKey[threadKey]; - if (current?.tabId === tabId) return state; + if ( + current && + previewMiniPlayerSourceKey(current.source) === previewMiniPlayerSourceKey(source) + ) { + return state; + } return { byThreadKey: { ...state.byThreadKey, [threadKey]: { - tabId, + source, position: current?.position ?? null, width: current?.width ?? null, }, @@ -53,11 +85,11 @@ export const usePreviewMiniPlayerStore = create()(( const { [threadKey]: _closed, ...byThreadKey } = state.byThreadKey; return { byThreadKey }; }), - move: (ref, tabId, position) => + move: (ref, sourceKey, position) => set((state) => { const threadKey = scopedThreadKey(ref); const current = state.byThreadKey[threadKey]; - if (!current || current.tabId !== tabId) return state; + if (!current || previewMiniPlayerSourceKey(current.source) !== sourceKey) return state; if (current.position?.x === position.x && current.position.y === position.y) return state; return { byThreadKey: { @@ -66,11 +98,17 @@ export const usePreviewMiniPlayerStore = create()(( }, }; }), - resize: (ref, tabId, width) => + resize: (ref, sourceKey, width) => set((state) => { const threadKey = scopedThreadKey(ref); const current = state.byThreadKey[threadKey]; - if (!current || current.tabId !== tabId || current.width === width) return state; + if ( + !current || + previewMiniPlayerSourceKey(current.source) !== sourceKey || + current.width === width + ) { + return state; + } return { byThreadKey: { ...state.byThreadKey, @@ -94,3 +132,12 @@ export function selectThreadPreviewMiniPlayer( if (!ref) return null; return byThreadKey[scopedThreadKey(ref)] ?? null; } + +/** The floating browser tab, or null when nothing floats or a device does. */ +export function selectThreadPreviewMiniPlayerTabId( + byThreadKey: Record, + ref: ScopedThreadRef | null | undefined, +): string | null { + const source = selectThreadPreviewMiniPlayer(byThreadKey, ref)?.source; + return source?.kind === "browser" ? source.tabId : null; +} diff --git a/docs/user/devices.md b/docs/user/devices.md index 7d1b91792ae3..d359d64d498d 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -18,6 +18,10 @@ device to boot it. The panel shows when you or an agent starts a device. Each device opens in its own tab. Use **+ → Device** to open another, and double-click a tab name or choose **Rename** from its context menu to rename it. Only the visible tab streams video; switching tabs keeps both devices running. +Choose **Float device over chat** in the toolbar to keep watching and tapping the +device in a small window while the right panel shows something else; drag the +window by its handle, resize it from any edge, and use **Open in right panel** +to bring it back. Turn off the device hub in **Settings → Integrations → Devices** to stop the helper processes; simulators and emulators keep running until you power them off. @@ -48,8 +52,10 @@ back from the device after a change. ## Agents and devices -When an agent opens a device, the panel opens in web and desktop clients connected -to the thread. Mobile clients show device activity in the thread timeline. Agents drive the device through the `agent-device` command line. T3 +When an agent opens a device, it floats over the chat in web and desktop clients +connected to the thread, the same way an agent-driven browser does. Turn off +**Auto-show floating preview** in **Settings → Integrations → Browser** to open a +right-panel tab instead. Mobile clients show device activity in the thread timeline. Agents drive the device through the `agent-device` command line. T3 Code installs and starts it only after **Agent device access** is enabled. iOS taps build a small test runner on first use, which takes a couple of minutes once per server. Restart an existing agent session after granting access so it From a2f7b0811dae95bc08f3d1f07cb71f72438570ac Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 11 Sep 2026 13:16:36 -0700 Subject: [PATCH 2/4] fix(web): clip the floating device at a phone-like corner radius Simulators and emulators stream a rectangular framebuffer with the display's rounded corners filled black, which showed as dark wedges in the floating player's 12px corners. The device player now clips at a radius scaled with its short side, with the hover pill inset to stay inside the curve; the browser player keeps its frame radius. Co-Authored-By: Claude Code --- .../preview/ThreadPreviewMiniPlayer.tsx | 19 ++++++++++++++++--- .../preview/previewMiniPlayerLayout.test.ts | 9 +++++++++ .../preview/previewMiniPlayerLayout.ts | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index bbd219bca3cb..626fc09a90af 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -34,9 +34,11 @@ import type { DeviceScreenSize } from "../device/deviceStream"; import { previewBridge } from "./previewBridge"; import { clampPreviewMiniPlayerPosition, + PREVIEW_MINI_PLAYER_CORNER_RADIUS, PREVIEW_MINI_PLAYER_WEBVIEW_Z_INDEX, type PreviewMiniPlayerFrame, resizePreviewMiniPlayer, + resolveDeviceMiniPlayerCornerRadius, resolveDeviceMiniPlayerSourceSize, resolvePreviewMiniPlayerFrame, resolvePreviewMiniPlayerSourceSize, @@ -56,7 +58,7 @@ interface Props { readonly bottomInset: number; } -const PREVIEW_MINI_PLAYER_CORNER_RADIUS = 12; +const frameCornerRadius = () => PREVIEW_MINI_PLAYER_CORNER_RADIUS; // Invisible grab zones straddling each edge; the cursor is the only affordance. const RESIZE_HANDLES: ReadonlyArray<{ @@ -226,6 +228,7 @@ function DeviceMiniPlayer({ bottomInset={bottomInset} label="Floating device preview" onOpenInPanel={openInPanel} + cornerRadius={resolveDeviceMiniPlayerCornerRadius} > {() => ( // The stream is DOM, so it takes the band the browser's native webview would. @@ -262,6 +265,7 @@ function MiniPlayerShell({ label, onOpenInPanel, pillActions, + cornerRadius = frameCornerRadius, children, }: { readonly threadRef: ScopedThreadRef; @@ -271,6 +275,8 @@ function MiniPlayerShell({ readonly label: string; readonly onOpenInPanel: () => void; readonly pillActions?: ReactNode; + /** The clip radius for a given frame; the pill stays inside the curve. */ + readonly cornerRadius?: (frame: PreviewMiniPlayerSize) => number; readonly children: (frame: PreviewMiniPlayerFrame) => ReactNode; }) { const containerRef = useRef(null); @@ -287,6 +293,10 @@ function MiniPlayerShell({ }) : null; + const radius = frame ? cornerRadius(frame) : PREVIEW_MINI_PLAYER_CORNER_RADIUS; + // Inside a wide curve the default 8px inset would land on the clipped-away corner. + const pillInset = Math.max(8, Math.round(radius * 0.55)); + const close = () => { usePreviewMiniPlayerStore.getState().close(threadRef); }; @@ -375,10 +385,13 @@ function MiniPlayerShell({ top: frame.y, width: frame.width, height: frame.height, - borderRadius: PREVIEW_MINI_PLAYER_CORNER_RADIUS, + borderRadius: radius, }} > -
+