diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 536b536a34..f692409a47 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { readFileSync } from "node:fs"; import { initSandboxRuntimeModular } from "./init"; +import { collectRuntimeTimelinePayload } from "./timeline"; import { TYPEGPU_PRESENT_HEARTBEAT_MS } from "./adapters/typegpu"; import { WebAudioTransport } from "./webAudioTransport"; import type { RuntimeTimelineLike } from "./types"; @@ -2199,6 +2200,83 @@ describe("initSandboxRuntimeModular", () => { expect(pipVideo.style.visibility).toBe("hidden"); }); + // The clip manifest the studio timeline draws and the runtime that actually + // plays the media must resolve a media element's absolute start through the + // same function. When they disagree the editor is a lie: the clip is drawn at + // one time and plays at another, and nothing fails. + // + // Both cases below are read from the SAME DOM the runtime just initialised, + // so the expected value is whatever playback uses, never a hardcoded number. + it("reports the same media start in the clip manifest as the runtime plays at", () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + document.body.appendChild(root); + + const host = document.createElement("div"); + host.setAttribute("data-composition-id", "scene-pip"); + host.setAttribute("data-composition-file", "compositions/pip.html"); + host.setAttribute("data-start", "45.40"); + host.setAttribute("data-duration", "7.06"); + root.appendChild(host); + + // Legacy root-global authoring: data-start is already absolute, so the host + // offset must NOT be added on top of it. + const pipVideo = document.createElement("video"); + pipVideo.id = "pip"; + pipVideo.setAttribute("data-start", "45.40"); + pipVideo.setAttribute("data-hf-media-start-basis", "global"); + pipVideo.setAttribute("data-duration", "7.06"); + host.appendChild(pipVideo); + + window.__timelines = { + main: createMockTimeline(60), + "scene-pip": createMockTimeline(7.06), + }; + initSandboxRuntimeModular(); + + const runtimeStart = window.__hfResolveMediaStartSeconds?.(pipVideo); + const manifestClip = collectRuntimeTimelinePayload({ canonicalFps: 30 }).clips.find( + (clip) => clip.id === "pip", + ); + expect(runtimeStart).toBeCloseTo(45.4); + expect(manifestClip?.start).toBeCloseTo(runtimeStart!); + }); + + it("keeps a composition-local media clip in the manifest at the time it plays", () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + document.body.appendChild(root); + + // No data-duration and no registered timeline anywhere, so the media window + // IS the composition's duration — which is what the second, attribute-only + // start derivation used to get wrong. + const host = document.createElement("div"); + host.setAttribute("data-composition-id", "scene-a"); + host.setAttribute("data-start", "10"); + root.appendChild(host); + + const nested = document.createElement("video"); + nested.id = "nested"; + nested.setAttribute("data-start", "2"); + nested.setAttribute("data-duration", "3"); + host.appendChild(nested); + + window.__timelines = {}; + initSandboxRuntimeModular(); + + const runtimeStart = window.__hfResolveMediaStartSeconds?.(nested); + const manifestClip = collectRuntimeTimelinePayload({ canonicalFps: 30 }).clips.find( + (clip) => clip.id === "nested", + ); + expect(runtimeStart).toBeCloseTo(12); + expect(manifestClip?.start).toBeCloseTo(runtimeStart!); + expect(manifestClip?.duration).toBeCloseTo(3); + }); + it("shows auto-injected video at host time, not at t=0", () => { const root = document.createElement("div"); root.setAttribute("data-composition-id", "main"); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 658c227559..b1ccb39811 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -65,12 +65,8 @@ import type { PlayerAPI } from "../core.types"; import { swallow } from "./diagnostics"; import { shouldAttemptPeriodicTimelineBind } from "./timelineRebindPolicy"; import { installStudioCustomEase } from "./customEase"; -import { parseNumeric } from "./startExpression"; -import { parseStrictFiniteTimingNumber } from "./playbackRate"; -import { - MEDIA_START_BASIS_ATTR, - resolveAbsoluteMediaStartSeconds as resolveAuthoredMediaStartSeconds, -} from "../mediaTiming"; +import { parseStrictFiniteTimingNumber, resolveMediaElementDurationSeconds } from "./playbackRate"; +import { MEDIA_START_BASIS_ATTR } from "../mediaTiming"; import { clearRuntimeData, setRuntimeData, @@ -712,24 +708,11 @@ export function initSandboxRuntimeModular(): void { return { compositionRoot, inheritedStart, inheritedDuration }; }; - const resolveAbsoluteMediaStartSeconds = (element: Element): number => { - const context = resolveMediaCompositionContext(element); - const inheritedStart = context.inheritedStart ?? 0; - const authoredStart = parseNumeric(element.getAttribute("data-start")); - if ( - element.hasAttribute("data-hf-auto-start") || - authoredStart == null || - inheritedStart <= 0 - ) { - return resolveStartForElement(element, inheritedStart); - } - - return resolveAuthoredMediaStartSeconds({ - authoredStart, - hostStart: inheritedStart, - basis: element.getAttribute(MEDIA_START_BASIS_ATTR), - }); - }; + // Single owner: `createRuntimeStartTimeResolver` (startResolver.ts). The clip + // manifest resolves media starts through the same method, so what the studio + // draws and what the transport plays cannot drift apart. + const resolveAbsoluteMediaStartSeconds = (element: Element): number => + timingResolverFor(true).resolveMediaStartForElement(element); window.__hfResolveMediaStartSeconds = resolveAbsoluteMediaStartSeconds; runtimeCleanupCallbacks.push(() => { @@ -832,17 +815,6 @@ export function initSandboxRuntimeModular(): void { }; }; - const resolveMediaElementDurationSeconds = (node: HTMLMediaElement): number | null => { - const declaredDuration = parseStrictFiniteTimingNumber(node.getAttribute("data-duration")); - if (declaredDuration != null && declaredDuration > 0) { - return declaredDuration; - } - if (Number.isFinite(node.duration)) { - return resolveNaturalMediaTimelineDuration(node, node.duration); - } - return null; - }; - // Scope 3 of 3 (see `withTimingResolver`). Every media element resolves its // composition ancestry here, and this runs on every transport tick via // `getSafeTimelineDurationSeconds`, so it is the heaviest consumer of the @@ -3282,7 +3254,7 @@ export function initSandboxRuntimeModular(): void { for (const rawEl of audioEls) { if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue; if (isSilencedByHidden(rawEl)) continue; - const start = Number.parseFloat(rawEl.dataset.start ?? ""); + const start = resolveAbsoluteMediaStartSeconds(rawEl); const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration); const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity; const mediaStart = readElementPlaybackStart(rawEl); @@ -3373,7 +3345,8 @@ export function initSandboxRuntimeModular(): void { for (const el of mediaEls) { if (!(el instanceof HTMLMediaElement)) continue; if (!el.isConnected) continue; - const start = Number.parseFloat(el.dataset.start ?? ""); + if (!el.hasAttribute("data-start")) continue; + const start = resolveAbsoluteMediaStartSeconds(el); if (!Number.isFinite(start)) continue; const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration); const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity; @@ -3403,7 +3376,7 @@ export function initSandboxRuntimeModular(): void { for (const rawEl of audioEls) { if (!(rawEl instanceof HTMLMediaElement) || !rawEl.isConnected) continue; if (isSilencedByHidden(rawEl)) continue; - const compStart = Number.parseFloat(rawEl.dataset.start ?? ""); + const compStart = resolveAbsoluteMediaStartSeconds(rawEl); if (!Number.isFinite(compStart)) continue; const mediaStart = readElementPlaybackStart(rawEl); const volumeAttr = Number.parseFloat(rawEl.dataset.volume ?? ""); diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts index 00840ae62e..da3d9650a2 100644 --- a/packages/core/src/runtime/mediaVolumeEnvelope.test.ts +++ b/packages/core/src/runtime/mediaVolumeEnvelope.test.ts @@ -201,6 +201,41 @@ describe("probeAndCacheElementVolume", () => { expect(interpolateVolumeGain(envelope, 0.5)).toBeCloseTo(1, 5); expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5); }); + it("uses the clip's absolute start, not its composition-local data-start", () => { + // Same fade as above, but the clip lives in a host composition that begins + // at t=2, so its `data-start="1"` means timeline t=3. Reading the attribute + // directly probed [1,2] — a window the clip is not even on screen for — and + // rebased the envelope 2s early. + const host = document.createElement("div"); + host.setAttribute("data-composition-id", "scene-a"); + host.dataset.start = "2"; + document.body.append(host); + const audio = document.createElement("audio"); + audio.dataset.start = "1"; + audio.dataset.duration = "1"; + audio.dataset.volume = "1"; + host.append(audio); + + const timeline = { + totalTime(next?: number) { + if (next !== undefined) { + // 0.05s linear fade-in at the clip's real start (timeline t=3). + audio.volume = Math.max(0, Math.min(1, (next - 3) / 0.05)); + } + return 0; + }, + }; + const cache = new WeakMap(); + + probeAndCacheElementVolume(audio, timeline, 4, cache); + + const envelope = cache.get(audio); + if (!envelope) throw new Error("Expected a cached envelope"); + expect(interpolateVolumeGain(envelope, 0)).toBeCloseTo(0, 5); + expect(interpolateVolumeGain(envelope, 0.05)).toBeCloseTo(1, 5); + expect(interpolateVolumeGain(envelope, 1)).toBeCloseTo(1, 5); + }); + it("keeps a fade that starts from an above-unity authored gain", () => { const audio = document.createElement("audio"); audio.dataset.start = "0"; diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.ts b/packages/core/src/runtime/mediaVolumeEnvelope.ts index 9c654cbf17..7099d04fa7 100644 --- a/packages/core/src/runtime/mediaVolumeEnvelope.ts +++ b/packages/core/src/runtime/mediaVolumeEnvelope.ts @@ -1,6 +1,7 @@ import type { RuntimeTimelineLike } from "./types"; import { clampAudioGain, withUnclampedVolume } from "../audioGain.js"; import { parseStrictFiniteTimingNumber } from "./playbackRate"; +import { createRuntimeStartTimeResolver } from "./startResolver"; /** * Shared volume-automation utilities used by both the renderer (offline PCM @@ -102,11 +103,26 @@ function parseVolumeNumber(value: string | undefined): number | undefined { return Number.isFinite(parsed) ? parsed : undefined; } +interface ProbeWindow { + start: number; + end: number; + staticVolume: number; +} + function resolveVolumeProbeWindow( el: HTMLAudioElement | HTMLVideoElement, compositionDuration: number, -): { start: number; end: number; staticVolume: number } { - const start = parseStrictFiniteTimingNumber(el.dataset.start) ?? 0; +): ProbeWindow { + // Probe samples are stamped with ROOT-timeline seek times, and + // `normaliseEnvelope` rebases them by this start — so it has to be the same + // absolute start the transport plays the clip at. Reading `data-start` + // directly gave a composition-local value, which put a nested clip's whole + // envelope at the wrong origin. + const start = createRuntimeStartTimeResolver({ + timelineRegistry: (window as Window & { __timelines?: Record }) + .__timelines, + includeAuthoredTimingAttrs: true, + }).resolveMediaStartForElement(el); const endAttr = parseStrictFiniteTimingNumber(el.dataset.end) ?? undefined; const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration) ?? undefined; let end = compositionDuration; @@ -135,8 +151,25 @@ export function probeElementVolumeKeyframes( compositionDuration: number, sampleFps: number, ): VolumeKeyframe[] | null { - const { start, end, staticVolume } = resolveVolumeProbeWindow(el, compositionDuration); + return probeKeyframesInWindow( + el, + seekTimeline, + compositionDuration, + sampleFps, + resolveVolumeProbeWindow(el, compositionDuration), + ); +} +/** Sampling half of the probe, given an already-resolved window. Split out so + * `probeAndCacheElementVolume` resolves that window ONCE and reuses it for the + * envelope rebase, instead of deriving the same start twice per element. */ +function probeKeyframesInWindow( + el: HTMLAudioElement | HTMLVideoElement, + seekTimeline: (t: number) => void, + compositionDuration: number, + sampleFps: number, + { start, end, staticVolume }: ProbeWindow, +): VolumeKeyframe[] | null { const step = 1 / Math.min(60, Math.max(1, sampleFps)); const sampleStart = Math.max(0, start); const sampleEnd = Math.min(compositionDuration, end); @@ -225,11 +258,11 @@ export function probeAndCacheElementVolume( : typeof timeline.seek === "function" ? Number(timeline.seek()) : 0; - const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60); + const probeWindow = resolveVolumeProbeWindow(mediaEl, compositionDuration); + const keyframes = probeKeyframesInWindow(mediaEl, seekFn, compositionDuration, 60, probeWindow); if (Number.isFinite(originalTime)) seekFn(originalTime); if (keyframes) { - const { start, staticVolume } = resolveVolumeProbeWindow(mediaEl, compositionDuration); - const envelope = normaliseEnvelope(keyframes, start, staticVolume); + const envelope = normaliseEnvelope(keyframes, probeWindow.start, probeWindow.staticVolume); if (envelope.length > 0) cache.set(mediaEl, envelope); } } diff --git a/packages/core/src/runtime/playbackRate.ts b/packages/core/src/runtime/playbackRate.ts index 4d016860b3..57fff775a8 100644 --- a/packages/core/src/runtime/playbackRate.ts +++ b/packages/core/src/runtime/playbackRate.ts @@ -39,6 +39,23 @@ export function resolveNaturalMediaTimelineDuration( ); } +/** + * How long a media element occupies the timeline: an explicit `data-duration` + * trim if authored, otherwise the natural source length adjusted for playback + * start and rate. `null` when the source has not reported a duration yet. + * + * Single owner for the media-window scan run by BOTH the runtime's duration + * floor and the clip manifest. + */ +export function resolveMediaElementDurationSeconds( + el: Pick & { duration: number }, +): number | null { + const declaredDuration = parseStrictFiniteTimingNumber(el.getAttribute("data-duration")); + if (declaredDuration != null && declaredDuration > 0) return declaredDuration; + if (Number.isFinite(el.duration)) return resolveNaturalMediaTimelineDuration(el, el.duration); + return null; +} + export function resolveNaturalMediaTimelineDurationFromValues( sourceDuration: number, mediaStart: number, diff --git a/packages/core/src/runtime/startResolver.ts b/packages/core/src/runtime/startResolver.ts index f9ac701b46..e2b3e78fba 100644 --- a/packages/core/src/runtime/startResolver.ts +++ b/packages/core/src/runtime/startResolver.ts @@ -1,9 +1,16 @@ import type { RuntimeTimelineLike } from "./types"; import { swallow } from "./diagnostics"; import { resolveAuthoredTimingWindow } from "./authoredTiming"; -import { readElementPlaybackRate } from "./media"; -import { readMediaStart } from "./playbackRate"; +// Straight from playbackRate, not through media.ts's re-export: media.ts +// imports mediaVolumeEnvelope, which needs this resolver, and the round trip +// would be an import cycle. +import { + parseStrictFiniteTimingNumber, + readElementPlaybackRate, + readMediaStart, +} from "./playbackRate"; import { parseStartExpression } from "./startExpression"; +import { MEDIA_START_BASIS_ATTR, resolveAbsoluteMediaStartSeconds } from "../mediaTiming"; export function createRuntimeStartTimeResolver(params: { timelineRegistry?: Record; @@ -18,6 +25,7 @@ export function createRuntimeStartTimeResolver(params: { }): { resolveStartForElement: (element: Element, fallback?: number) => number; resolveDurationForElement: (element: Element) => number | null; + resolveMediaStartForElement: (element: Element) => number; } { const timelineRegistry = params.timelineRegistry ?? {}; const includeAuthoredTimingAttrs = params.includeAuthoredTimingAttrs ?? false; @@ -175,10 +183,39 @@ export function createRuntimeStartTimeResolver(params: { } }; + /** + * The ONE owner of "when does this media element start on the root timeline". + * + * A media element is not a plain timed clip: `data-hf-media-start-basis` + * decides whether its `data-start` is composition-local (the default, so the + * host offset is added) or a legacy root-global timestamp (already absolute, + * so adding the host offset double-counts it). Anything that derives a media + * start from attributes — the clip manifest, the visibility pass, the media + * cache, WebAudio scheduling — must come through here, or the timeline the + * editor draws stops matching the timeline that plays. + */ + const resolveMediaStartForElement = (element: Element): number => { + const compositionRoot = element.closest("[data-composition-id]"); + const hostStart = compositionRoot ? resolveStartForElementInternal(compositionRoot, 0) : 0; + const authoredStart = parseStrictFiniteTimingNumber(element.getAttribute("data-start")); + // No literal start (absent, or a `data-start="intro + 2"` reference), an + // auto-injected start, or a host at t=0 — nothing for the basis to + // disambiguate, so the ordinary start resolution is already correct. + if (element.hasAttribute("data-hf-auto-start") || authoredStart == null || hostStart <= 0) { + return resolveStartForElementInternal(element, hostStart); + } + return resolveAbsoluteMediaStartSeconds({ + authoredStart, + hostStart, + basis: element.getAttribute(MEDIA_START_BASIS_ATTR), + }); + }; + return { resolveStartForElement: (element: Element, fallback = 0) => resolveStartForElementInternal(element, Math.max(0, fallback)), resolveDurationForElement: (element: Element) => resolveDurationForElement(element), + resolveMediaStartForElement, }; } diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts index 2d82ef4857..21797d08aa 100644 --- a/packages/core/src/runtime/timeline.ts +++ b/packages/core/src/runtime/timeline.ts @@ -8,7 +8,11 @@ import { stableClipId } from "./clipTree"; import { resolveAuthoredTimingWindow } from "./authoredTiming"; import { swallow } from "./diagnostics"; import { readElementPlaybackRate, readElementPlaybackStart } from "./media"; -import { parseStrictFiniteTimingNumber, resolveNaturalMediaTimelineDuration } from "./playbackRate"; +import { + parseStrictFiniteTimingNumber, + resolveMediaElementDurationSeconds, + resolveNaturalMediaTimelineDuration, +} from "./playbackRate"; import { resolveCssStackingContextId } from "./stackingContext"; import { createRuntimeStartTimeResolver } from "./startResolver"; import { isSceneLikeCompositionId } from "../slideshow/index.js"; @@ -182,18 +186,6 @@ export function collectRuntimeTimelinePayload(params: { return null; } }; - const resolveMediaElementDurationSeconds = ( - mediaEl: HTMLVideoElement | HTMLAudioElement, - ): number | null => { - const declaredDuration = parseNum(mediaEl.getAttribute("data-duration")); - if (declaredDuration != null && declaredDuration > 0) { - return declaredDuration; - } - if (Number.isFinite(mediaEl.duration)) { - return resolveNaturalMediaTimelineDuration(mediaEl, mediaEl.duration); - } - return null; - }; const resolveMediaWindowEndSeconds = (): number | null => { const mediaNodes = Array.from( document.querySelectorAll("video[data-start], audio[data-start]"), @@ -201,9 +193,7 @@ export function collectRuntimeTimelinePayload(params: { if (mediaNodes.length === 0) return null; let maxWindowEndSeconds = 0; for (const mediaNode of mediaNodes) { - const start = !mediaNode.hasAttribute("data-hf-auto-start") - ? Math.max(0, Number(mediaNode.getAttribute("data-start") ?? 0) || 0) - : startResolver.resolveStartForElement(mediaNode, 0); + const start = startResolver.resolveMediaStartForElement(mediaNode); if (!Number.isFinite(start)) continue; const duration = resolveMediaElementDurationSeconds(mediaNode); if (duration == null || duration <= 0) continue; @@ -356,10 +346,11 @@ export function collectRuntimeTimelinePayload(params: { if (["SCRIPT", "STYLE", "LINK", "META", "TEMPLATE", "NOSCRIPT"].includes(node.tagName)) continue; const compositionContext = resolveNearestCompositionContext(node, root); - const start = startResolver.resolveStartForElement( - node, - compositionContext.inheritedStart ?? 0, - ); + const tag = node.tagName.toLowerCase(); + const start = + tag === "video" || tag === "audio" + ? startResolver.resolveMediaStartForElement(node) + : startResolver.resolveStartForElement(node, compositionContext.inheritedStart ?? 0); const nodeCompositionId = node.getAttribute("data-composition-id"); let duration = parseElementDurationAttr(node); if (duration == null && nodeCompositionId && nodeCompositionId !== rootCompositionId) { @@ -383,7 +374,6 @@ export function collectRuntimeTimelinePayload(params: { if (duration <= 0) continue; const end = start + duration; maxEnd = Math.max(maxEnd, end); - const tag = node.tagName.toLowerCase(); const kind: RuntimeTimelineClip["kind"] = nodeCompositionId && nodeCompositionId !== rootCompositionId ? "composition"