fix(web): preserve the active pointer in the flat Device view

apps/web/src/components/device/DeviceStreamView.browser.test.tsx

@@ -0,0 +1,166 @@
+import { EnvironmentId } from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+
+import { act, render, stubGlobal } from "../../../test/render";
+
+vi.mock(import("~/state/device"), async (original) => ({
+  ...(await original()),
+  useDeviceHubAccess: () => access,
+}));
+vi.mock(import("./DevicePhoneViewport"), () => ({
+  DevicePhoneViewport: () => <></>,
+}));
+import { DeviceStreamView } from "./DeviceStreamView";
+
+const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };
+function setup() {
+  const touches: unknown[] = [];
+  // Select MJPEG without starting a WebCodecs decoder. Input uses the real stream client.
+  const decoder = stubGlobal("VideoDecoder", undefined);
+  Reflect.deleteProperty(globalThis, "VideoDecoder");
+  const fetch = stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
+  // The client opens its input socket only after the MJPEG prime resolves.
+  type InputSocket = { onmessage?: (event: { data: ArrayBuffer }) => void };
+  let open!: (socket: InputSocket) => void;
+  const opened = new Promise<InputSocket>((resolve) => {
+    open = resolve;
+  });
+  const socket = stubGlobal(
+    "WebSocket",
+    class {
+      static OPEN = 1;
+      readyState = 1;
+      constructor() {
+        open(this);
+      }
+      onmessage?: (event: { data: ArrayBuffer }) => void;
+      send(message: string | Uint8Array) {
+        if (typeof message !== "string" && message[0] === 0x03) {
+          touches.push(JSON.parse(new TextDecoder().decode(message.subarray(1))));
+        }
+      }
+      close() {}
+    },
+  );
+  return {
+    [Symbol.dispose]() {
+      socket[Symbol.dispose]();
+      fetch[Symbol.dispose]();
+      decoder[Symbol.dispose]();
+    },
+    touches,
+    async configure() {
+      const json = new TextEncoder().encode(
+        JSON.stringify({ width: 400, height: 800, orientation: "portrait" }),
+      );
+      const packet = new Uint8Array(1 + json.length);
+      packet[0] = 0x82;
+      packet.set(json, 1);
+      const input = await opened;
+      input.onmessage?.({ data: Uint8Array.of(0x83).buffer });
+      input.onmessage?.({ data: packet.buffer });
+    },
+  };
+}
+
+function view() {
+  return (
+    <DeviceStreamView
+      hostId="local"
+      environmentId={EnvironmentId.make("test")}
+      deviceId="test"
+      platform="ios"
+      visible
+      allowPhoneView={false}
+    />
+  );
+}
+
+// router: false because the device view renders no links and never navigates.
+const options = { router: false } as const;
+
+function flatPointer(container: HTMLElement) {
+  const surface = container.querySelector("canvas")!.parentElement!;
+  // Synthetic IDs exercise concurrent touches independently of the browser's capture state.
+  surface.setPointerCapture = () => {};
+  const rect = new DOMRect(0, 0, 400, 800);
+  surface.getBoundingClientRect = () => rect;
+  return {
+    send(type: string, pointerId: number, x = 0.25, y = 0.25) {
+      surface.dispatchEvent(
+        new PointerEvent(type, {
+          bubbles: true,
+          pointerId,
+          pointerType: "touch",
+          clientX: rect.left + rect.width * x,
+          clientY: rect.top + rect.height * y,
+        }),
+      );
+    },
+  };
+}
+
+it("keeps the first flat pointer active when a second finger moves and lifts", async () => {
+  using input = setup();
+  const { configure, touches } = input;
+  await using screen = await render(view(), options);
+  await act(async () => configure());
+  const { send } = flatPointer(screen.container);
+  send("pointerdown", 1);
+  send("pointerdown", 2, 0.75, 0.75);
+  send("pointermove", 2, 0.875, 0.875);
+  send("pointerup", 2);
+  send("pointercancel", 2);
+  send("pointermove", 1, 0.5, 0.5);
+  send("pointerup", 1, 0.75, 0.75);
+  send("pointermove", 1);
+  expect(touches).toEqual([
+    { type: "begin", x: 0.25, y: 0.25 },
+    { type: "move", x: 0.5, y: 0.5 },
+    { type: "end", x: 0.75, y: 0.75 },
+  ]);
+});
+
+it("ignores a flat pointer that goes down before input connects", async () => {
+  using input = setup();
+  const { configure, touches } = input;
+  await using screen = await render(view(), options);
+  const { send } = flatPointer(screen.container);
+  send("pointerdown", 1);
+  await act(async () => configure());
+  send("pointerdown", 2, 0.5, 0.5);
+  send("pointerup", 2, 0.75, 0.75);
+  expect(touches).toEqual([
+    { type: "begin", x: 0.5, y: 0.5 },
+    { type: "end", x: 0.75, y: 0.75 },
+  ]);
+});
+
+it.each(["pointerup", "pointercancel", "lostpointercapture"])(
+  "ends the owned flat pointer exactly once on %s, then accepts a new pointer",
+  async (ending) => {
+    using input = setup();
+    const { configure, touches } = input;
+    await using screen = await render(view(), options);
+    await act(async () => configure());
+    const { send } = flatPointer(screen.container);
+    send("pointerdown", 7);
+    send("pointermove", 7, 0.5, 0.5);
+    send("lostpointercapture", 8, 0, 0);
+    // Capture loss reports its own point; the end must still use the last move.
+    const at = ending === "lostpointercapture" ? 0 : 0.5;
+    send(ending, 7, at, at);
+    send("lostpointercapture", 7, 0, 0);
+    send("pointerup", 7);
+    send("pointermove", 7);
+    send("pointerdown", 9);
+    send("pointerup", 9);
+    expect(touches).toEqual([
+      { type: "begin", x: 0.25, y: 0.25 },
+      { type: "move", x: 0.5, y: 0.5 },
+      { type: "end", x: 0.5, y: 0.5 },
+      { type: "begin", x: 0.25, y: 0.25 },
+      { type: "end", x: 0.25, y: 0.25 },
+    ]);
+  },
+);

apps/web/src/components/device/DeviceStreamView.tsx

@@ -117,6 +117,7 @@ export function DeviceStreamView(props: {
   const access = useDeviceHubAccess(props.environmentId, props.hostId);
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
   const clientRef = useRef<DeviceStreamClient | null>(null);
+  const activePointer = useRef<{ id: number; x: number; y: number } | null>(null);
   const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
   const [hasFrame, setHasFrame] = useState(false);
   const [detail, setDetail] = useState<string | undefined>(undefined);
@@ -185,6 +186,7 @@ export function DeviceStreamView(props: {
     onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false });
     return () => {
       cancelPhoneInput();
+      activePointer.current = null;
       client.stop();
       clientRef.current = null;
       onHandle?.(null);
@@ -286,7 +288,6 @@ export function DeviceStreamView(props: {
     };
   }, [access, props.axOverlay, props.deviceId, props.platform, props.visible]);
 
-  const pointerActive = useRef(false);
   const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
     const rect = event.currentTarget.getBoundingClientRect();
     const x = (event.clientX - rect.left) / rect.width;
@@ -413,30 +414,40 @@ export function DeviceStreamView(props: {
           className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
           style={{ width: frame.width, height: frame.height }}
           onPointerDown={(event) => {
-            if (event.button !== 0) return;
+            // iOS drops touches sent before admission, so an early pointer would never begin.
+            if (event.button !== 0 || !inputState.connected || activePointer.current !== null) {
+              return;
+            }
             event.currentTarget.setPointerCapture(event.pointerId);
             (event.currentTarget.parentElement as HTMLElement | null)?.focus();
-            pointerActive.current = true;
             const { x, y } = normalizedPoint(event);
+            activePointer.current = { id: event.pointerId, x, y };
             clientRef.current?.sendTouch("begin", x, y);
           }}
           onPointerMove={(event) => {
-            if (!pointerActive.current) return;
+            if (activePointer.current?.id !== event.pointerId) return;
             const { x, y } = normalizedPoint(event);
+            activePointer.current = { id: event.pointerId, x, y };
             clientRef.current?.sendTouch("move", x, y);
           }}
           onPointerUp={(event) => {
-            if (!pointerActive.current) return;
-            pointerActive.current = false;
+            if (activePointer.current?.id !== event.pointerId) return;
+            activePointer.current = null;
             const { x, y } = normalizedPoint(event);
             clientRef.current?.sendTouch("end", x, y);
           }}
           onPointerCancel={(event) => {
-            if (!pointerActive.current) return;
-            pointerActive.current = false;
+            if (activePointer.current?.id !== event.pointerId) return;
+            activePointer.current = null;
             const { x, y } = normalizedPoint(event);
             clientRef.current?.sendTouch("end", x, y);
           }}
+          onLostPointerCapture={(event) => {
+            const pointer = activePointer.current;
+            if (pointer?.id !== event.pointerId) return;
+            activePointer.current = null;
+            clientRef.current?.sendTouch("end", pointer.x, pointer.y);
+          }}
         >
           <canvas
             ref={canvasRef}