fix(runtime): resolve media clip start once for playback, manifest, and audio

packages/core/src/runtime/init.test.ts

@@ -2,6 +2,7 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { readFileSync } from "node:fs";
 import { initSandboxRuntimeModular } from "./init";
+import { collectRuntimeTimelinePayload } from "./timeline";
 import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu";
 import { WebAudioTransport } from "./webAudioTransport";
 import type { RuntimeTimelineLike } from "./types";
@@ -2199,6 +2200,83 @@ describe("initSandboxRuntimeModular", () => {
     expect(pipVideo.style.visibility).toBe("hidden");
   });
 
+  // The clip manifest the studio timeline draws and the runtime that actually
+  // plays the media must resolve a media element's absolute start through the
+  // same function. When they disagree the editor is a lie: the clip is drawn at
+  // one time and plays at another, and nothing fails.
+  //
+  // Both cases below are read from the SAME DOM the runtime just initialised,
+  // so the expected value is whatever playback uses, never a hardcoded number.
+  it("reports the same media start in the clip manifest as the runtime plays at", () => {
+    const root = document.createElement("div");
+    root.setAttribute("data-composition-id", "main");
+    root.setAttribute("data-root", "true");
+    root.setAttribute("data-start", "0");
+    document.body.appendChild(root);
+
+    const host = document.createElement("div");
+    host.setAttribute("data-composition-id", "scene-pip");
+    host.setAttribute("data-composition-file", "compositions/pip.html");
+    host.setAttribute("data-start", "45.40");
+    host.setAttribute("data-duration", "7.06");
+    root.appendChild(host);
+
+    // Legacy root-global authoring: data-start is already absolute, so the host
+    // offset must NOT be added on top of it.
+    const pipVideo = document.createElement("video");
+    pipVideo.id = "pip";
+    pipVideo.setAttribute("data-start", "45.40");
+    pipVideo.setAttribute("data-hf-media-start-basis", "global");
+    pipVideo.setAttribute("data-duration", "7.06");
+    host.appendChild(pipVideo);
+
+    window.__timelines = {
+      main: createMockTimeline(60),
+      "scene-pip": createMockTimeline(7.06),
+    };
+    initSandboxRuntimeModular();
+
+    const runtimeStart = window.__hfResolveMediaStartSeconds?.(pipVideo);
+    const manifestClip = collectRuntimeTimelinePayload({ canonicalFps: 30 }).clips.find(
+      (clip) => clip.id === "pip",
+    );
+    expect(runtimeStart).toBeCloseTo(45.4);
+    expect(manifestClip?.start).toBeCloseTo(runtimeStart!);
+  });
+
+  it("keeps a composition-local media clip in the manifest at the time it plays", () => {
+    const root = document.createElement("div");
+    root.setAttribute("data-composition-id", "main");
+    root.setAttribute("data-root", "true");
+    root.setAttribute("data-start", "0");
+    document.body.appendChild(root);
+
+    // No data-duration and no registered timeline anywhere, so the media window
+    // IS the composition's duration — which is what the second, attribute-only
+    // start derivation used to get wrong.
+    const host = document.createElement("div");
+    host.setAttribute("data-composition-id", "scene-a");
+    host.setAttribute("data-start", "10");
+    root.appendChild(host);
+
+    const nested = document.createElement("video");
+    nested.id = "nested";
+    nested.setAttribute("data-start", "2");
+    nested.setAttribute("data-duration", "3");
+    host.appendChild(nested);
+
+    window.__timelines = {};
+    initSandboxRuntimeModular();
+
+    const runtimeStart = window.__hfResolveMediaStartSeconds?.(nested);
+    const manifestClip = collectRuntimeTimelinePayload({ canonicalFps: 30 }).clips.find(
+      (clip) => clip.id === "nested",
+    );
+    expect(runtimeStart).toBeCloseTo(12);
+    expect(manifestClip?.start).toBeCloseTo(runtimeStart!);
+    expect(manifestClip?.duration).toBeCloseTo(3);
+  });
+
   it("shows auto-injected video at host time, not at t=0", () => {
     const root = document.createElement("div");
     root.setAttribute("data-composition-id", "main");

packages/core/src/runtime/init.ts

@@ -65,12 +65,8 @@ import type { PlayerAPI } from "../core.types";
 import { swallow } from "./diagnostics";
 import { shouldAttemptPeriodicTimelineBind } from "./timelineRebindPolicy";
 import { installStudioCustomEase } from "./customEase";
-import { parseNumeric } from "./startExpression";
-import { parseStrictFiniteTimingNumber } from "./playbackRate";
-import {
-  MEDIA_START_BASIS_ATTR,
-  resolveAbsoluteMediaStartSeconds as resolveAuthoredMediaStartSeconds,
-} from "../mediaTiming";
+import { parseStrictFiniteTimingNumber, resolveMediaElementDurationSeconds } from "./playbackRate";
+import { MEDIA_START_BASIS_ATTR } from "../mediaTiming";
 import {
   clearRuntimeData,
   setRuntimeData,
@@ -712,24 +708,11 @@ export function initSandboxRuntimeModular(): void {
     return { compositionRoot, inheritedStart, inheritedDuration };
   };
 
-  const resolveAbsoluteMediaStartSeconds = (element: Element): number => {
-    const context = resolveMediaCompositionContext(element);
-    const inheritedStart = context.inheritedStart ?? 0;
-    const authoredStart = parseNumeric(element.getAttribute("data-start"));
-    if (
-      element.hasAttribute("data-hf-auto-start") ||
-      authoredStart == null ||
-      inheritedStart <= 0
-    ) {
-      return resolveStartForElement(element, inheritedStart);
-    }
-
-    return resolveAuthoredMediaStartSeconds({
-      authoredStart,
-      hostStart: inheritedStart,
-      basis: element.getAttribute(MEDIA_START_BASIS_ATTR),
-    });
-  };
+  // Single owner: `createRuntimeStartTimeResolver` (startResolver.ts). The clip
+  // manifest resolves media starts through the same method, so what the studio
+  // draws and what the transport plays cannot drift apart.
+  const resolveAbsoluteMediaStartSeconds = (element: Element): number =>
+    timingResolverFor(true).resolveMediaStartForElement(element);
 
   window.__hfResolveMediaStartSeconds = resolveAbsoluteMediaStartSeconds;
   runtimeCleanupCallbacks.push(() => {
@@ -832,17 +815,6 @@ export function initSandboxRuntimeModular(): void {
     };
   };
 
-  const resolveMediaElementDurationSeconds = (node: HTMLMediaElement): number | null => {
-    const declaredDuration = parseStrictFiniteTimingNumber(node.getAttribute("data-duration"));
-    if (declaredDuration != null && declaredDuration > 0) {
-      return declaredDuration;
-    }
-    if (Number.isFinite(node.duration)) {
-      return resolveNaturalMediaTimelineDuration(node, node.duration);
-    }
-    return null;
-  };
-
   // Scope 3 of 3 (see `withTimingResolver`). Every media element resolves its
   // composition ancestry here, and this runs on every transport tick via
   // `getSafeTimelineDurationSeconds`, so it is the heaviest consumer of the
@@ -3282,7 +3254,7 @@ export function initSandboxRuntimeModular(): void {
           for (const rawEl of audioEls) {
             if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
             if (isSilencedByHidden(rawEl)) continue;
-            const start = Number.parseFloat(rawEl.dataset.start ?? "");
+            const start = resolveAbsoluteMediaStartSeconds(rawEl);
             const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);
             const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
             const mediaStart = readElementPlaybackStart(rawEl);
@@ -3373,7 +3345,8 @@ export function initSandboxRuntimeModular(): void {
     for (const el of mediaEls) {
       if (!(el instanceof HTMLMediaElement)) continue;
       if (!el.isConnected) continue;
-      const start = Number.parseFloat(el.dataset.start ?? "");
+      if (!el.hasAttribute("data-start")) continue;
+      const start = resolveAbsoluteMediaStartSeconds(el);
       if (!Number.isFinite(start)) continue;
       const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);
       const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
@@ -3403,7 +3376,7 @@ export function initSandboxRuntimeModular(): void {
     for (const rawEl of audioEls) {
       if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue;
       if (isSilencedByHidden(rawEl)) continue;
-      const compStart = Number.parseFloat(rawEl.dataset.start ?? "");
+      const compStart = resolveAbsoluteMediaStartSeconds(rawEl);
       if (!Number.isFinite(compStart)) continue;
       const mediaStart = readElementPlaybackStart(rawEl);
       const volumeAttr = Number.parseFloat(rawEl.dataset.volume ?? "");

packages/core/src/runtime/mediaVolumeEnvelope.test.ts

@@ -201,6 +201,41 @@ describe("probeAndCacheElementVolume", () => {
     expect(interpolateVolumeGain(envelope, 0.5)).toBeCloseTo(1, 5);
     expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5);
   });
+  it("uses the clip's absolute start, not its composition-local data-start", () => {
+    // Same fade as above, but the clip lives in a host composition that begins
+    // at t=2, so its `data-start="1"` means timeline t=3. Reading the attribute
+    // directly probed [1,2] — a window the clip is not even on screen for — and
+    // rebased the envelope 2s early.
+    const host = document.createElement("div");
+    host.setAttribute("data-composition-id", "scene-a");
+    host.dataset.start = "2";
+    document.body.append(host);
+    const audio = document.createElement("audio");
+    audio.dataset.start = "1";
+    audio.dataset.duration = "1";
+    audio.dataset.volume = "1";
+    host.append(audio);
+
+    const timeline = {
+      totalTime(next?: number) {
+        if (next !== undefined) {
+          // 0.05s linear fade-in at the clip's real start (timeline t=3).
+          audio.volume = Math.max(0, Math.min(1, (next - 3) / 0.05));
+        }
+        return 0;
+      },
+    };
+    const cache = new WeakMap<HTMLMediaElement, { time: number; volume: number }[]>();
+
+    probeAndCacheElementVolume(audio, timeline, 4, cache);
+
+    const envelope = cache.get(audio);
+    if (!envelope) throw new Error("Expected a cached envelope");
+    expect(interpolateVolumeGain(envelope, 0)).toBeCloseTo(0, 5);
+    expect(interpolateVolumeGain(envelope, 0.05)).toBeCloseTo(1, 5);
+    expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5);
+  });
+
   it("keeps a fade that starts from an above-unity authored gain", () => {
     const audio = document.createElement("audio");
     audio.dataset.start = "0";

packages/core/src/runtime/mediaVolumeEnvelope.ts

@@ -1,6 +1,7 @@
 import type { RuntimeTimelineLike } from "./types";
 import { clampAudioGain, withUnclampedVolume } from "../audioGain.js";
 import { parseStrictFiniteTimingNumber } from "./playbackRate";
+import { createRuntimeStartTimeResolver } from "./startResolver";
 
 /**
  * Shared volume-automation utilities used by both the renderer (offline PCM
@@ -102,11 +103,26 @@ function parseVolumeNumber(value: string | undefined): number | undefined {
   return Number.isFinite(parsed) ? parsed : undefined;
 }
 
+interface ProbeWindow {
+  start: number;
+  end: number;
+  staticVolume: number;
+}
+
 function resolveVolumeProbeWindow(
   el: HTMLAudioElement | HTMLVideoElement,
   compositionDuration: number,
-): { start: number; end: number; staticVolume: number } {
-  const start = parseStrictFiniteTimingNumber(el.dataset.start) ?? 0;
+): ProbeWindow {
+  // Probe samples are stamped with ROOT-timeline seek times, and
+  // `normaliseEnvelope` rebases them by this start — so it has to be the same
+  // absolute start the transport plays the clip at. Reading `data-start`
+  // directly gave a composition-local value, which put a nested clip's whole
+  // envelope at the wrong origin.
+  const start = createRuntimeStartTimeResolver({
+    timelineRegistry: (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> })
+      .__timelines,
+    includeAuthoredTimingAttrs: true,
+  }).resolveMediaStartForElement(el);
   const endAttr = parseStrictFiniteTimingNumber(el.dataset.end) ?? undefined;
   const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration) ?? undefined;
   let end = compositionDuration;
@@ -135,8 +151,25 @@ export function probeElementVolumeKeyframes(
   compositionDuration: number,
   sampleFps: number,
 ): VolumeKeyframe[] | null {
-  const { start, end, staticVolume } = resolveVolumeProbeWindow(el, compositionDuration);
+  return probeKeyframesInWindow(
+    el,
+    seekTimeline,
+    compositionDuration,
+    sampleFps,
+    resolveVolumeProbeWindow(el, compositionDuration),
+  );
+}
 
+/** Sampling half of the probe, given an already-resolved window. Split out so
+ *  `probeAndCacheElementVolume` resolves that window ONCE and reuses it for the
+ *  envelope rebase, instead of deriving the same start twice per element. */
+function probeKeyframesInWindow(
+  el: HTMLAudioElement | HTMLVideoElement,
+  seekTimeline: (t: number) => void,
+  compositionDuration: number,
+  sampleFps: number,
+  { start, end, staticVolume }: ProbeWindow,
+): VolumeKeyframe[] | null {
   const step = 1 / Math.min(60, Math.max(1, sampleFps));
   const sampleStart = Math.max(0, start);
   const sampleEnd = Math.min(compositionDuration, end);
@@ -225,11 +258,11 @@ export function probeAndCacheElementVolume(
       : typeof timeline.seek === "function"
         ? Number(timeline.seek())
         : 0;
-  const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60);
+  const probeWindow = resolveVolumeProbeWindow(mediaEl, compositionDuration);
+  const keyframes = probeKeyframesInWindow(mediaEl, seekFn, compositionDuration, 60, probeWindow);
   if (Number.isFinite(originalTime)) seekFn(originalTime);
   if (keyframes) {
-    const { start, staticVolume } = resolveVolumeProbeWindow(mediaEl, compositionDuration);
-    const envelope = normaliseEnvelope(keyframes, start, staticVolume);
+    const envelope = normaliseEnvelope(keyframes, probeWindow.start, probeWindow.staticVolume);
     if (envelope.length > 0) cache.set(mediaEl, envelope);
   }
 }

packages/core/src/runtime/playbackRate.ts

@@ -39,6 +39,23 @@ export function resolveNaturalMediaTimelineDuration(
   );
 }
 
+/**
+ * How long a media element occupies the timeline: an explicit `data-duration`
+ * trim if authored, otherwise the natural source length adjusted for playback
+ * start and rate. `null` when the source has not reported a duration yet.
+ *
+ * Single owner for the media-window scan run by BOTH the runtime's duration
+ * floor and the clip manifest.
+ */
+export function resolveMediaElementDurationSeconds(
+  el: Pick<Element, "getAttribute"> & { duration: number },
+): number | null {
+  const declaredDuration = parseStrictFiniteTimingNumber(el.getAttribute("data-duration"));
+  if (declaredDuration != null && declaredDuration > 0) return declaredDuration;
+  if (Number.isFinite(el.duration)) return resolveNaturalMediaTimelineDuration(el, el.duration);
+  return null;
+}
+
 export function resolveNaturalMediaTimelineDurationFromValues(
   sourceDuration: number,
   mediaStart: number,

packages/core/src/runtime/startResolver.ts

@@ -1,9 +1,16 @@
 import type { RuntimeTimelineLike } from "./types";
 import { swallow } from "./diagnostics";
 import { resolveAuthoredTimingWindow } from "./authoredTiming";
-import { readElementPlaybackRate } from "./media";
-import { readMediaStart } from "./playbackRate";
+// Straight from playbackRate, not through media.ts's re-export: media.ts
+// imports mediaVolumeEnvelope, which needs this resolver, and the round trip
+// would be an import cycle.
+import {
+  parseStrictFiniteTimingNumber,
+  readElementPlaybackRate,
+  readMediaStart,
+} from "./playbackRate";
 import { parseStartExpression } from "./startExpression";
+import { MEDIA_START_BASIS_ATTR, resolveAbsoluteMediaStartSeconds } from "../mediaTiming";
 
 export function createRuntimeStartTimeResolver(params: {
   timelineRegistry?: Record<string, RuntimeTimelineLike | undefined>;
@@ -18,6 +25,7 @@ export function createRuntimeStartTimeResolver(params: {
 }): {
   resolveStartForElement: (element: Element, fallback?: number) => number;
   resolveDurationForElement: (element: Element) => number | null;
+  resolveMediaStartForElement: (element: Element) => number;
 } {
   const timelineRegistry = params.timelineRegistry ?? {};
   const includeAuthoredTimingAttrs = params.includeAuthoredTimingAttrs ?? false;
@@ -175,10 +183,39 @@ export function createRuntimeStartTimeResolver(params: {
     }
   };
 
+  /**
+   * The ONE owner of "when does this media element start on the root timeline".
+   *
+   * A media element is not a plain timed clip: `data-hf-media-start-basis`
+   * decides whether its `data-start` is composition-local (the default, so the
+   * host offset is added) or a legacy root-global timestamp (already absolute,
+   * so adding the host offset double-counts it). Anything that derives a media
+   * start from attributes — the clip manifest, the visibility pass, the media
+   * cache, WebAudio scheduling — must come through here, or the timeline the
+   * editor draws stops matching the timeline that plays.
+   */
+  const resolveMediaStartForElement = (element: Element): number => {
+    const compositionRoot = element.closest("[data-composition-id]");
+    const hostStart = compositionRoot ? resolveStartForElementInternal(compositionRoot, 0) : 0;
+    const authoredStart = parseStrictFiniteTimingNumber(element.getAttribute("data-start"));
+    // No literal start (absent, or a `data-start="intro + 2"` reference), an
+    // auto-injected start, or a host at t=0 — nothing for the basis to
+    // disambiguate, so the ordinary start resolution is already correct.
+    if (element.hasAttribute("data-hf-auto-start") || authoredStart == null || hostStart <= 0) {
+      return resolveStartForElementInternal(element, hostStart);
+    }
+    return resolveAbsoluteMediaStartSeconds({
+      authoredStart,
+      hostStart,
+      basis: element.getAttribute(MEDIA_START_BASIS_ATTR),
+    });
+  };
+
   return {
     resolveStartForElement: (element: Element, fallback = 0) =>
       resolveStartForElementInternal(element, Math.max(0, fallback)),
     resolveDurationForElement: (element: Element) => resolveDurationForElement(element),
+    resolveMediaStartForElement,
   };
 }