fix(mobile): restore low-latency decoding for simulator streams

docs/internals/devices.md

@@ -105,10 +105,14 @@ binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over
 one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts)
 speaks both so one panel covers both platforms.
 
-Simulators encode H.264 High 5.1. Hardware decoders on some machines and all
-headless browsers reject that profile, and WebCodecs is secure-context only, so
-the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on
-iOS. Android has no MJPEG; there the panel reports that it cannot decode.
+Simulators encode H.264 High 5.1 without frame reordering, but the hub's fallback
+encoder can omit the SPS VUI that declares its reorder limits. Chromium's hardware
+decoder then retains frames for seconds despite `optimizeForLatency` and an empty
+`decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before
+configuring the decoder so hardware decoding can remain enabled. WebCodecs is
+secure-context only, so the viewer probes `isConfigSupported` and falls back to
+MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel
+reports that it cannot decode.
 
 serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
 AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so

packages/client-runtime/src/device/avcConfig.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { simulatorAvcConfig } from "./avcConfig.ts";
+
+// Captured from the hub before and after its encoder falls back from low-latency rate control.
+const lowLatency = Uint8Array.from([
+  1, 100, 0, 51, 255, 225, 0, 22, 39, 100, 0, 51, 172, 19, 20, 60, 4, 192, 20, 158, 106, 154, 129,
+  1, 1, 3, 194, 1, 8, 248, 1, 0, 4, 40, 238, 60, 176,
+]);
+const fallback = Uint8Array.from([
+  1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, 0, 4, 40,
+  238, 60, 176,
+]);
+const repaired = Uint8Array.from([
+  1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, 68, 35,
+  80, 1, 0, 4, 40, 238, 60, 176,
+]);
+
+describe("simulator AVC reorder metadata", () => {
+  it("adds zero reordering and one reference frame to the captured fallback SPS", () => {
+    expect(simulatorAvcConfig(fallback)).toEqual(repaired);
+    expect(simulatorAvcConfig(repaired)).toBe(repaired);
+  });
+
+  it("preserves existing VUI and leaves the original codec description untouched", () => {
+    const original = fallback.slice();
+    simulatorAvcConfig(fallback);
+    expect(fallback).toEqual(original);
+    expect(simulatorAvcConfig(lowLatency)).toBe(lowLatency);
+  });
+
+  it("repairs multiple SPS entries while preserving PPS and profile extensions", () => {
+    const extensions = [0xfd, 0xf8, 0xf8, 0];
+    const description = Uint8Array.from([
+      ...fallback.subarray(0, 5),
+      0xe2,
+      ...fallback.subarray(6, 20),
+      ...lowLatency.subarray(6, 30),
+      ...fallback.subarray(20),
+      ...extensions,
+    ]);
+    expect(simulatorAvcConfig(description)).toEqual(
+      Uint8Array.from([
+        ...description.subarray(0, 6),
+        ...repaired.subarray(6, 25),
+        ...lowLatency.subarray(6, 30),
+        ...fallback.subarray(20),
+        ...extensions,
+      ]),
+    );
+  });
+
+  it("passes through every truncated prefix of the captured configurations", () => {
+    for (const config of [fallback, lowLatency]) {
+      for (let length = 0; length < config.length; length++) {
+        const truncated = config.subarray(0, length);
+        expect(simulatorAvcConfig(truncated)).toBe(truncated);
+      }
+    }
+  });
+
+  it("does not modify unsupported profiles, scaling lists, or invalid parameter-set lengths", () => {
+    const unsupportedProfile = fallback.slice();
+    unsupportedProfile[9] = 110;
+    const scalingLists = fallback.slice();
+    scalingLists[12] = scalingLists[12]! | 1;
+    const invalidSpsLength = fallback.slice();
+    invalidSpsLength[6] = 0xff;
+    const invalidPpsLength = fallback.slice();
+    invalidPpsLength[21] = 0xff;
+    for (const config of [unsupportedProfile, scalingLists, invalidSpsLength, invalidPpsLength]) {
+      expect(simulatorAvcConfig(config)).toBe(config);
+    }
+  });
+
+  it("requires valid SPS trailing bits before appending VUI metadata", () => {
+    const invalidPadding = fallback.slice();
+    invalidPadding[19] = 0;
+    expect(simulatorAvcConfig(invalidPadding)).toBe(invalidPadding);
+  });
+
+  it("passes through invalid NAL headers and unterminated Exp-Golomb values", () => {
+    const forbiddenBit = fallback.slice();
+    forbiddenBit[8] = forbiddenBit[8]! | 0x80;
+    expect(simulatorAvcConfig(forbiddenBit)).toBe(forbiddenBit);
+    const unterminated = fallback.slice();
+    unterminated.fill(0, 12, 20);
+    expect(simulatorAvcConfig(unterminated)).toBe(unterminated);
+    const overlong = Uint8Array.from([
+      ...fallback.subarray(0, 6),
+      0,
+      13,
+      39,
+      100,
+      0,
+      51,
+      0,
+      0,
+      3,
+      0,
+      0,
+      3,
+      0,
+      0,
+      0,
+      ...fallback.subarray(20),
+    ]);
+    expect(simulatorAvcConfig(overlong)).toBe(overlong);
+  });
+
+  it("bounds parsing of oversized configurations and SPS data", () => {
+    const oversizedConfig = new Uint8Array(64 * 1024 + 1);
+    oversizedConfig.set(fallback);
+    expect(simulatorAvcConfig(oversizedConfig)).toBe(oversizedConfig);
+    const oversizedSps = new Uint8Array(6 + 2 + 4097 + 7);
+    oversizedSps.set(fallback.subarray(0, 6));
+    oversizedSps.set([0x10, 1], 6);
+    oversizedSps.set(fallback.subarray(8, 20), 8);
+    oversizedSps.set(fallback.subarray(20), 8 + 4097);
+    expect(simulatorAvcConfig(oversizedSps)).toBe(oversizedSps);
+  });
+});

packages/client-runtime/src/device/avcConfig.ts

@@ -0,0 +1,141 @@
+const MAX_CONFIG_BYTES = 64 * 1024;
+const MAX_SPS_BYTES = 4096;
+
+const unsignedGolomb = (value: number) => {
+  const bits = (value + 1).toString(2);
+  return "0".repeat(bits.length - 1) + bits;
+};
+
+function withReorderLimits(sps: Uint8Array): Uint8Array {
+  if (sps.length < 4 || sps.length > MAX_SPS_BYTES || (sps[0]! & 0x9f) !== 7) return sps;
+  const rbsp: number[] = [];
+  let zeros = 0;
+  for (let index = 1; index < sps.length; index++) {
+    const byte = sps[index]!;
+    if (zeros >= 2 && byte === 3) {
+      if (index + 1 >= sps.length || sps[index + 1]! > 3) return sps;
+      zeros = 0;
+      continue;
+    }
+    rbsp.push(byte);
+    zeros = byte === 0 ? zeros + 1 : 0;
+  }
+  const bits = rbsp.map((byte) => byte.toString(2).padStart(8, "0")).join("");
+  let position = 0;
+  const read = (count: number) => {
+    if (position + count > bits.length) throw new Error("Truncated H.264 SPS");
+    const value = Number.parseInt(bits.slice(position, position + count), 2);
+    position += count;
+    return value;
+  };
+  const readUnsignedGolomb = () => {
+    let leadingZeros = 0;
+    while (read(1) === 0) {
+      if (++leadingZeros > 31) throw new Error("Invalid H.264 Exp-Golomb value");
+    }
+    return 2 ** leadingZeros - 1 + (leadingZeros === 0 ? 0 : read(leadingZeros));
+  };
+
+  try {
+    // Only handle the hub's 8-bit 4:2:0 High profile without scaling lists.
+    // Unknown encoder configurations pass through rather than guessing their layout.
+    if (read(8) !== 100) return sps;
+    read(16);
+    if (readUnsignedGolomb() > 31 || readUnsignedGolomb() !== 1) return sps;
+    if (readUnsignedGolomb() !== 0 || readUnsignedGolomb() !== 0) return sps;
+    read(1);
+    if (read(1) !== 0) return sps;
+    if (readUnsignedGolomb() > 12) return sps;
+    const pictureOrderCountType = readUnsignedGolomb();
+    if (pictureOrderCountType === 0) {
+      if (readUnsignedGolomb() > 12) return sps;
+    } else if (pictureOrderCountType === 1) {
+      read(1);
+      readUnsignedGolomb();
+      readUnsignedGolomb();
+      const cycleLength = readUnsignedGolomb();
+      if (cycleLength > 255) return sps;
+      for (let index = 0; index < cycleLength; index++) readUnsignedGolomb();
+    } else if (pictureOrderCountType !== 2) return sps;
+    const maxReferenceFrames = readUnsignedGolomb();
+    if (maxReferenceFrames > 16) return sps;
+    read(1);
+    if (readUnsignedGolomb() > 65535 || readUnsignedGolomb() > 65535) return sps;
+    if (read(1) === 0) read(1);
+    read(1);
+    if (read(1) !== 0) {
+      for (let index = 0; index < 4; index++) readUnsignedGolomb();
+    }
+    const vuiPosition = position;
+    if (read(1) !== 0) return sps;
+    // With no VUI, only rbsp_stop_one_bit and byte padding may follow.
+    if (!/^10{0,7}$/.test(bits.slice(position))) return sps;
+
+    // serve-sim disables frame reordering. Its fallback encoder omits the VUI,
+    // causing Chromium to infer a large reorder queue despite optimizeForLatency.
+    // Preserve the SPS and add VUI defaults with max_num_reorder_frames = 0.
+    // The flag bits enable VUI, omit its eight optional sections, and enable
+    // bitstream restrictions and motion_vectors_over_pic_boundaries_flag.
+    const restricted =
+      bits.slice(0, vuiPosition) +
+      "10000000011" +
+      [2, 1, 16, 16, 0, maxReferenceFrames].map(unsignedGolomb).join("") +
+      "1";
+    const padded = restricted.padEnd(Math.ceil(restricted.length / 8) * 8, "0");
+    const output = [sps[0]!];
+    zeros = 0;
+    for (let index = 0; index < padded.length; index += 8) {
+      const byte = Number.parseInt(padded.slice(index, index + 8), 2);
+      if (zeros >= 2 && byte <= 3) {
+        output.push(3);
+        zeros = 0;
+      }
+      output.push(byte);
+      zeros = byte === 0 ? zeros + 1 : 0;
+    }
+    return Uint8Array.from(output);
+  } catch {
+    return sps;
+  }
+}
+
+/** Repair missing reorder metadata only for serve-sim's non-reordering AVCC feed. */
+export function simulatorAvcConfig(description: Uint8Array): Uint8Array {
+  if (description.length < 7 || description.length > MAX_CONFIG_BYTES || description[0] !== 1)
+    return description;
+  const parameterSets = description[5]! & 0x1f;
+  if (parameterSets === 0) return description;
+  const parts: Uint8Array[] = [description.subarray(0, 6)];
+  let position = 6;
+  let changed = false;
+  for (let index = 0; index < parameterSets; index++) {
+    if (position + 2 > description.length) return description;
+    const length = description[position]! * 256 + description[position + 1]!;
+    position += 2;
+    if (length === 0 || position + length > description.length) return description;
+    const sps = description.subarray(position, position + length);
+    const repaired = withReorderLimits(sps);
+    changed ||= repaired !== sps;
+    parts.push(Uint8Array.of(repaired.length >>> 8, repaired.length & 0xff), repaired);
+    position += length;
+  }
+  const suffix = position;
+  if (position >= description.length) return description;
+  const pictureParameterSets = description[position++]!;
+  for (let index = 0; index < pictureParameterSets; index++) {
+    if (position + 2 > description.length) return description;
+    const length = description[position]! * 256 + description[position + 1]!;
+    position += 2;
+    if (length === 0 || position + length > description.length) return description;
+    position += length;
+  }
+  if (!changed) return description;
+  parts.push(description.subarray(suffix));
+  const output = new Uint8Array(parts.reduce((length, part) => length + part.length, 0));
+  position = 0;
+  for (const part of parts) {
+    output.set(part, position);
+    position += part.length;
+  }
+  return output;
+}

packages/client-runtime/src/device/stream.test.ts

@@ -323,6 +323,36 @@ describe("shared device stream readiness and recovery", () => {
     vi.unstubAllGlobals();
   });
 
+  it("repairs the hub's fallback configuration without disabling hardware decoding", async () => {
+    const { client, videoBodies, Decoder } = recoveryFixture("ios", false);
+    try {
+      client.start();
+      await vi.advanceTimersByTimeAsync(0);
+      videoBodies[0]!.enqueue(
+        Uint8Array.from(
+          envelope(
+            1,
+            [
+              1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1,
+              0, 4, 40, 238, 60, 176,
+            ],
+          ),
+        ),
+      );
+      await vi.advanceTimersByTimeAsync(0);
+      expect(Decoder.isConfigSupported).toHaveBeenCalledExactlyOnceWith({
+        codec: "avc1.640033",
+        description: Uint8Array.from([
+          1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208,
+          68, 35, 80, 1, 0, 4, 40, 238, 60, 176,
+        ]),
+        optimizeForLatency: true,
+      });
+    } finally {
+      client.stop();
+    }
+  });
+
   it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
     const { client, videoBodies, drawImage, decodedFrame } = recoveryFixture("ios", false);
     let resolveBitmap!: (bitmap: ImageBitmap) => void;

packages/client-runtime/src/device/stream.ts

@@ -10,6 +10,7 @@ import {
 import * as Schema from "effect/Schema";
 import * as Option from "effect/Option";
 import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
+import { simulatorAvcConfig } from "./avcConfig.ts";
 import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
 import type { DevicePlatform } from "@t3tools/contracts";
 
@@ -759,7 +760,10 @@ export function createDeviceStreamClient(
             case "description": {
               awaitingKeyframe = true;
               const configured = await configureDecoder(
-                { codec: avcCodecString(chunk.payload), description: chunk.payload },
+                {
+                  codec: avcCodecString(chunk.payload),
+                  description: simulatorAvcConfig(chunk.payload),
+                },
                 isCurrent,
               );
               if (!isCurrent()) return;