From 40264c34edb511d5b09841f219396209ab4d9877 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Wed, 21 Oct 2026 02:39:25 -0500 Subject: [PATCH] fix(mobile): restore low-latency decoding for simulator streams - Add missing H.264 reorder metadata to fallback AVCC configurations - Preserve hardware decoding and document the simulator encoder behavior --- docs/internals/devices.md | 12 +- .../src/device/avcConfig.test.ts | 122 +++++++++++++++ .../client-runtime/src/device/avcConfig.ts | 141 ++++++++++++++++++ .../client-runtime/src/device/stream.test.ts | 30 ++++ packages/client-runtime/src/device/stream.ts | 6 +- 5 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 packages/client-runtime/src/device/avcConfig.test.ts create mode 100644 packages/client-runtime/src/device/avcConfig.ts diff --git a/docs/internals/devices.md b/docs/internals/devices.md index b0d03ab76a64..6e91307f09d4 100644 --- a/docs/internals/devices.md +++ b/docs/internals/devices.md @@ -105,10 +105,14 @@ binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts) speaks both so one panel covers both platforms. -Simulators encode H.264 High 5.1. Hardware decoders on some machines and all -headless browsers reject that profile, and WebCodecs is secure-context only, so -the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on -iOS. Android has no MJPEG; there the panel reports that it cannot decode. +Simulators encode H.264 High 5.1 without frame reordering, but the hub's fallback +encoder can omit the SPS VUI that declares its reorder limits. Chromium's hardware +decoder then retains frames for seconds despite `optimizeForLatency` and an empty +`decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before +configuring the decoder so hardware decoding can remain enabled. WebCodecs is +secure-context only, so the viewer probes `isConfigSupported` and falls back to +MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel +reports that it cannot decode. serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so diff --git a/packages/client-runtime/src/device/avcConfig.test.ts b/packages/client-runtime/src/device/avcConfig.test.ts new file mode 100644 index 000000000000..4b2ff9906f3f --- /dev/null +++ b/packages/client-runtime/src/device/avcConfig.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { simulatorAvcConfig } from "./avcConfig.ts"; + +// Captured from the hub before and after its encoder falls back from low-latency rate control. +const lowLatency = Uint8Array.from([ + 1, 100, 0, 51, 255, 225, 0, 22, 39, 100, 0, 51, 172, 19, 20, 60, 4, 192, 20, 158, 106, 154, 129, + 1, 1, 3, 194, 1, 8, 248, 1, 0, 4, 40, 238, 60, 176, +]); +const fallback = Uint8Array.from([ + 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, 0, 4, 40, + 238, 60, 176, +]); +const repaired = Uint8Array.from([ + 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, 68, 35, + 80, 1, 0, 4, 40, 238, 60, 176, +]); + +describe("simulator AVC reorder metadata", () => { + it("adds zero reordering and one reference frame to the captured fallback SPS", () => { + expect(simulatorAvcConfig(fallback)).toEqual(repaired); + expect(simulatorAvcConfig(repaired)).toBe(repaired); + }); + + it("preserves existing VUI and leaves the original codec description untouched", () => { + const original = fallback.slice(); + simulatorAvcConfig(fallback); + expect(fallback).toEqual(original); + expect(simulatorAvcConfig(lowLatency)).toBe(lowLatency); + }); + + it("repairs multiple SPS entries while preserving PPS and profile extensions", () => { + const extensions = [0xfd, 0xf8, 0xf8, 0]; + const description = Uint8Array.from([ + ...fallback.subarray(0, 5), + 0xe2, + ...fallback.subarray(6, 20), + ...lowLatency.subarray(6, 30), + ...fallback.subarray(20), + ...extensions, + ]); + expect(simulatorAvcConfig(description)).toEqual( + Uint8Array.from([ + ...description.subarray(0, 6), + ...repaired.subarray(6, 25), + ...lowLatency.subarray(6, 30), + ...fallback.subarray(20), + ...extensions, + ]), + ); + }); + + it("passes through every truncated prefix of the captured configurations", () => { + for (const config of [fallback, lowLatency]) { + for (let length = 0; length < config.length; length++) { + const truncated = config.subarray(0, length); + expect(simulatorAvcConfig(truncated)).toBe(truncated); + } + } + }); + + it("does not modify unsupported profiles, scaling lists, or invalid parameter-set lengths", () => { + const unsupportedProfile = fallback.slice(); + unsupportedProfile[9] = 110; + const scalingLists = fallback.slice(); + scalingLists[12] = scalingLists[12]! | 1; + const invalidSpsLength = fallback.slice(); + invalidSpsLength[6] = 0xff; + const invalidPpsLength = fallback.slice(); + invalidPpsLength[21] = 0xff; + for (const config of [unsupportedProfile, scalingLists, invalidSpsLength, invalidPpsLength]) { + expect(simulatorAvcConfig(config)).toBe(config); + } + }); + + it("requires valid SPS trailing bits before appending VUI metadata", () => { + const invalidPadding = fallback.slice(); + invalidPadding[19] = 0; + expect(simulatorAvcConfig(invalidPadding)).toBe(invalidPadding); + }); + + it("passes through invalid NAL headers and unterminated Exp-Golomb values", () => { + const forbiddenBit = fallback.slice(); + forbiddenBit[8] = forbiddenBit[8]! | 0x80; + expect(simulatorAvcConfig(forbiddenBit)).toBe(forbiddenBit); + const unterminated = fallback.slice(); + unterminated.fill(0, 12, 20); + expect(simulatorAvcConfig(unterminated)).toBe(unterminated); + const overlong = Uint8Array.from([ + ...fallback.subarray(0, 6), + 0, + 13, + 39, + 100, + 0, + 51, + 0, + 0, + 3, + 0, + 0, + 3, + 0, + 0, + 0, + ...fallback.subarray(20), + ]); + expect(simulatorAvcConfig(overlong)).toBe(overlong); + }); + + it("bounds parsing of oversized configurations and SPS data", () => { + const oversizedConfig = new Uint8Array(64 * 1024 + 1); + oversizedConfig.set(fallback); + expect(simulatorAvcConfig(oversizedConfig)).toBe(oversizedConfig); + const oversizedSps = new Uint8Array(6 + 2 + 4097 + 7); + oversizedSps.set(fallback.subarray(0, 6)); + oversizedSps.set([0x10, 1], 6); + oversizedSps.set(fallback.subarray(8, 20), 8); + oversizedSps.set(fallback.subarray(20), 8 + 4097); + expect(simulatorAvcConfig(oversizedSps)).toBe(oversizedSps); + }); +}); diff --git a/packages/client-runtime/src/device/avcConfig.ts b/packages/client-runtime/src/device/avcConfig.ts new file mode 100644 index 000000000000..1c600635799c --- /dev/null +++ b/packages/client-runtime/src/device/avcConfig.ts @@ -0,0 +1,141 @@ +const MAX_CONFIG_BYTES = 64 * 1024; +const MAX_SPS_BYTES = 4096; + +const unsignedGolomb = (value: number) => { + const bits = (value + 1).toString(2); + return "0".repeat(bits.length - 1) + bits; +}; + +function withReorderLimits(sps: Uint8Array): Uint8Array { + if (sps.length < 4 || sps.length > MAX_SPS_BYTES || (sps[0]! & 0x9f) !== 7) return sps; + const rbsp: number[] = []; + let zeros = 0; + for (let index = 1; index < sps.length; index++) { + const byte = sps[index]!; + if (zeros >= 2 && byte === 3) { + if (index + 1 >= sps.length || sps[index + 1]! > 3) return sps; + zeros = 0; + continue; + } + rbsp.push(byte); + zeros = byte === 0 ? zeros + 1 : 0; + } + const bits = rbsp.map((byte) => byte.toString(2).padStart(8, "0")).join(""); + let position = 0; + const read = (count: number) => { + if (position + count > bits.length) throw new Error("Truncated H.264 SPS"); + const value = Number.parseInt(bits.slice(position, position + count), 2); + position += count; + return value; + }; + const readUnsignedGolomb = () => { + let leadingZeros = 0; + while (read(1) === 0) { + if (++leadingZeros > 31) throw new Error("Invalid H.264 Exp-Golomb value"); + } + return 2 ** leadingZeros - 1 + (leadingZeros === 0 ? 0 : read(leadingZeros)); + }; + + try { + // Only handle the hub's 8-bit 4:2:0 High profile without scaling lists. + // Unknown encoder configurations pass through rather than guessing their layout. + if (read(8) !== 100) return sps; + read(16); + if (readUnsignedGolomb() > 31 || readUnsignedGolomb() !== 1) return sps; + if (readUnsignedGolomb() !== 0 || readUnsignedGolomb() !== 0) return sps; + read(1); + if (read(1) !== 0) return sps; + if (readUnsignedGolomb() > 12) return sps; + const pictureOrderCountType = readUnsignedGolomb(); + if (pictureOrderCountType === 0) { + if (readUnsignedGolomb() > 12) return sps; + } else if (pictureOrderCountType === 1) { + read(1); + readUnsignedGolomb(); + readUnsignedGolomb(); + const cycleLength = readUnsignedGolomb(); + if (cycleLength > 255) return sps; + for (let index = 0; index < cycleLength; index++) readUnsignedGolomb(); + } else if (pictureOrderCountType !== 2) return sps; + const maxReferenceFrames = readUnsignedGolomb(); + if (maxReferenceFrames > 16) return sps; + read(1); + if (readUnsignedGolomb() > 65535 || readUnsignedGolomb() > 65535) return sps; + if (read(1) === 0) read(1); + read(1); + if (read(1) !== 0) { + for (let index = 0; index < 4; index++) readUnsignedGolomb(); + } + const vuiPosition = position; + if (read(1) !== 0) return sps; + // With no VUI, only rbsp_stop_one_bit and byte padding may follow. + if (!/^10{0,7}$/.test(bits.slice(position))) return sps; + + // serve-sim disables frame reordering. Its fallback encoder omits the VUI, + // causing Chromium to infer a large reorder queue despite optimizeForLatency. + // Preserve the SPS and add VUI defaults with max_num_reorder_frames = 0. + // The flag bits enable VUI, omit its eight optional sections, and enable + // bitstream restrictions and motion_vectors_over_pic_boundaries_flag. + const restricted = + bits.slice(0, vuiPosition) + + "10000000011" + + [2, 1, 16, 16, 0, maxReferenceFrames].map(unsignedGolomb).join("") + + "1"; + const padded = restricted.padEnd(Math.ceil(restricted.length / 8) * 8, "0"); + const output = [sps[0]!]; + zeros = 0; + for (let index = 0; index < padded.length; index += 8) { + const byte = Number.parseInt(padded.slice(index, index + 8), 2); + if (zeros >= 2 && byte <= 3) { + output.push(3); + zeros = 0; + } + output.push(byte); + zeros = byte === 0 ? zeros + 1 : 0; + } + return Uint8Array.from(output); + } catch { + return sps; + } +} + +/** Repair missing reorder metadata only for serve-sim's non-reordering AVCC feed. */ +export function simulatorAvcConfig(description: Uint8Array): Uint8Array { + if (description.length < 7 || description.length > MAX_CONFIG_BYTES || description[0] !== 1) + return description; + const parameterSets = description[5]! & 0x1f; + if (parameterSets === 0) return description; + const parts: Uint8Array[] = [description.subarray(0, 6)]; + let position = 6; + let changed = false; + for (let index = 0; index < parameterSets; index++) { + if (position + 2 > description.length) return description; + const length = description[position]! * 256 + description[position + 1]!; + position += 2; + if (length === 0 || position + length > description.length) return description; + const sps = description.subarray(position, position + length); + const repaired = withReorderLimits(sps); + changed ||= repaired !== sps; + parts.push(Uint8Array.of(repaired.length >>> 8, repaired.length & 0xff), repaired); + position += length; + } + const suffix = position; + if (position >= description.length) return description; + const pictureParameterSets = description[position++]!; + for (let index = 0; index < pictureParameterSets; index++) { + if (position + 2 > description.length) return description; + const length = description[position]! * 256 + description[position + 1]!; + position += 2; + if (length === 0 || position + length > description.length) return description; + position += length; + } + if (!changed) return description; + parts.push(description.subarray(suffix)); + const output = new Uint8Array(parts.reduce((length, part) => length + part.length, 0)); + position = 0; + for (const part of parts) { + output.set(part, position); + position += part.length; + } + return output; +} diff --git a/packages/client-runtime/src/device/stream.test.ts b/packages/client-runtime/src/device/stream.test.ts index 5ffdddb26f2e..7da96cb3d75b 100644 --- a/packages/client-runtime/src/device/stream.test.ts +++ b/packages/client-runtime/src/device/stream.test.ts @@ -323,6 +323,36 @@ describe("shared device stream readiness and recovery", () => { vi.unstubAllGlobals(); }); + it("repairs the hub's fallback configuration without disabling hardware decoding", async () => { + const { client, videoBodies, Decoder } = recoveryFixture("ios", false); + try { + client.start(); + await vi.advanceTimersByTimeAsync(0); + videoBodies[0]!.enqueue( + Uint8Array.from( + envelope( + 1, + [ + 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, + 0, 4, 40, 238, 60, 176, + ], + ), + ), + ); + await vi.advanceTimersByTimeAsync(0); + expect(Decoder.isConfigSupported).toHaveBeenCalledExactlyOnceWith({ + codec: "avc1.640033", + description: Uint8Array.from([ + 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, + 68, 35, 80, 1, 0, 4, 40, 238, 60, 176, + ]), + optimizeForLatency: true, + }); + } finally { + client.stop(); + } + }); + it("does not repaint a delayed JPEG seed over newer decoded video", async () => { const { client, videoBodies, drawImage, decodedFrame } = recoveryFixture("ios", false); let resolveBitmap!: (bitmap: ImageBitmap) => void; diff --git a/packages/client-runtime/src/device/stream.ts b/packages/client-runtime/src/device/stream.ts index 9318c50fe691..04388e3429de 100644 --- a/packages/client-runtime/src/device/stream.ts +++ b/packages/client-runtime/src/device/stream.ts @@ -10,6 +10,7 @@ import { import * as Schema from "effect/Schema"; import * as Option from "effect/Option"; import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts"; +import { simulatorAvcConfig } from "./avcConfig.ts"; import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts"; import type { DevicePlatform } from "@t3tools/contracts"; @@ -759,7 +760,10 @@ export function createDeviceStreamClient( case "description": { awaitingKeyframe = true; const configured = await configureDecoder( - { codec: avcCodecString(chunk.payload), description: chunk.payload }, + { + codec: avcCodecString(chunk.payload), + description: simulatorAvcConfig(chunk.payload), + }, isCurrent, ); if (!isCurrent()) return;