From f6f1c760c92d5168cbf30f5f9ac6d7041f762aab Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Fri, 5 Dec 2025 21:36:40 -0700 Subject: [PATCH] Add frame timing and pacing for video playback Implements proper frame timing to prevent frame burst at playback start. Frames are now rendered based on their timestamps relative to playback start time, with late frame dropping to maintain smooth playback. Changes: - Add frame timing system with playback start time tracking - Wrap VideoFrames with timestamp metadata using proper wrapper objects - Use requestAnimationFrame-based scheduling with frame dropping - Add memory leak protection (max 100 timestamp entries) - Standardize worker messages to use consistent object format - Add diagnostic test page to measure frame burst behavior The frame burst test page demonstrates the fix and provides metrics on frame rendering intervals. --- lib/playback/backend.ts | 2 +- lib/playback/worker/video.ts | 95 ++- .../frame-burst-test/frame-burst-test.html | 550 ++++++++++++++++++ samples/index.html | 6 + 4 files changed, 646 insertions(+), 7 deletions(-) create mode 100644 samples/frame-burst-test/frame-burst-test.html diff --git a/lib/playback/backend.ts b/lib/playback/backend.ts index f0ad3cb..223125e 100644 --- a/lib/playback/backend.ts +++ b/lib/playback/backend.ts @@ -114,7 +114,7 @@ export default class Backend { private on(e: MessageEvent) { const msg = e.data - if (msg === "waitingforkeyframe") { + if (msg.type === "waitingforkeyframe") { this.#eventTarget.dispatchEvent(new Event("waitingforkeyframe")) } } diff --git a/lib/playback/worker/video.ts b/lib/playback/worker/video.ts index f433a57..b9846a0 100644 --- a/lib/playback/worker/video.ts +++ b/lib/playback/worker/video.ts @@ -18,18 +18,32 @@ interface DecoderConfig { optimizeForLatency?: boolean } +// Wrapper for VideoFrame with original timestamp +interface FrameWithTimestamp { + frame: VideoFrame + originalTimestamp: number +} + export class Renderer { #canvas: OffscreenCanvas #timeline: Component #decoder!: VideoDecoder - #queue: TransformStream + #queue: TransformStream #decoderConfig?: DecoderConfig #waitingForKeyframe: boolean = true #paused: boolean #hasSentWaitingForKeyFrameEvent: boolean = false + // Frame timing for proper playback rate + #playbackStartTime: number | null = null + #firstFrameTimestamp: number | null = null + + // Map to store original timestamps for frames (with memory leak protection) + #frameTimestamps: Map = new Map() + #MAX_TIMESTAMP_MAP_SIZE = 100 // Prevent memory leaks + constructor(config: Message.ConfigVideo, timeline: Component) { this.#canvas = config.canvas this.#timeline = timeline @@ -49,20 +63,57 @@ export class Renderer { console.error(err) }) this.#waitingForKeyframe = true + // Reset timing on pause so next play starts fresh + this.#playbackStartTime = null + this.#firstFrameTimestamp = null } play() { this.#paused = false + // Reset timing on play to start fresh + this.#playbackStartTime = null + this.#firstFrameTimestamp = null } async #run() { const reader = this.#timeline.frames.pipeThrough(this.#queue).getReader() + for (;;) { - const { value: frame, done } = await reader.read() + const { value: frameWithTimestamp, done } = await reader.read() if (this.#paused) continue if (done) break + const frame = frameWithTimestamp.frame + const frameTimestampMs = frameWithTimestamp.originalTimestamp + + // Initialize timing on first frame + if (this.#firstFrameTimestamp === null || this.#playbackStartTime === null) { + this.#firstFrameTimestamp = frameTimestampMs + this.#playbackStartTime = performance.now() + } + + // Calculate when this frame should be displayed + const frameOffsetMs = frameTimestampMs - this.#firstFrameTimestamp + const targetDisplayTime = this.#playbackStartTime + frameOffsetMs + + // Use requestAnimationFrame with timestamp for smoother playback self.requestAnimationFrame(() => { + // Check if still not paused (could have paused during wait) + if (this.#paused) { + frame.close() + return + } + + // Check if we should skip this frame (too late) + const now = performance.now() + const lateness = now - targetDisplayTime + + // If we're more than 50ms late, skip this frame + if (lateness > 50) { + frame.close() + return + } + this.#canvas.width = frame.displayWidth this.#canvas.height = frame.displayHeight @@ -75,10 +126,26 @@ export class Renderer { } } - #start(controller: TransformStreamDefaultController) { + #start(controller: TransformStreamDefaultController) { this.#decoder = new VideoDecoder({ output: (frame: VideoFrame) => { - controller.enqueue(frame) + // Retrieve the original timestamp from our map + const originalTimestamp = this.#frameTimestamps.get(frame.timestamp) + if (originalTimestamp !== undefined) { + // Wrap frame with original timestamp + controller.enqueue({ + frame: frame, + originalTimestamp: originalTimestamp, + }) + // Clean up the map entry + this.#frameTimestamps.delete(frame.timestamp) + } else { + // Fallback: use the frame's timestamp converted to milliseconds + controller.enqueue({ + frame: frame, + originalTimestamp: frame.timestamp / 1000, + }) + } }, error: console.error, }) @@ -140,7 +207,7 @@ export class Renderer { if (this.#waitingForKeyframe && !frame.sample.is_sync) { console.warn("Skipping non-keyframe until a keyframe is found.") if (!this.#hasSentWaitingForKeyFrameEvent) { - self.postMessage("waitingforkeyframe") + self.postMessage({ type: "waitingforkeyframe" }) this.#hasSentWaitingForKeyFrameEvent = true } return @@ -152,10 +219,26 @@ export class Renderer { this.#hasSentWaitingForKeyFrameEvent = false } + // Calculate timestamp in seconds for the chunk (standard WebCodecs unit) + const timestampSeconds = frame.sample.dts / frame.track.timescale + + // Store the original timestamp (in milliseconds) so we can retrieve it later + const timestampMs = timestampSeconds * 1000 + + // Prevent memory leak: if map gets too large, clear oldest entries + if (this.#frameTimestamps.size >= this.#MAX_TIMESTAMP_MAP_SIZE) { + const firstKey = this.#frameTimestamps.keys().next().value + if (firstKey !== undefined) { + this.#frameTimestamps.delete(firstKey) + } + } + + this.#frameTimestamps.set(timestampSeconds, timestampMs) + const chunk = new EncodedVideoChunk({ type: frame.sample.is_sync ? "key" : "delta", data: frame.sample.data, - timestamp: frame.sample.dts / frame.track.timescale, + timestamp: timestampSeconds, }) this.#decoder.decode(chunk) diff --git a/samples/frame-burst-test/frame-burst-test.html b/samples/frame-burst-test/frame-burst-test.html new file mode 100644 index 0000000..43bd804 --- /dev/null +++ b/samples/frame-burst-test/frame-burst-test.html @@ -0,0 +1,550 @@ + + + + + MoQ Player - Frame Burst Test + + + + +
+

πŸ“Š MoQ Player - Frame Burst Test

+ +
+

Test Objective

+

This test demonstrates and measures the "frame burst" problem at playback start.

+

Expected behavior: Frames should be rendered at a consistent rate (e.g., ~33ms for 30fps).

+

Actual behavior (bug): Initial frames are rendered rapidly in a burst (<5ms intervals), then playback normalizes.

+

Test criteria: FAIL if more than 5 frames are rendered with <10ms intervals at the start.

+
+ + + +
+ + +
+ +
+
+ +
+
+

πŸ“ˆ Real-time Metrics

+
+ Total Frames: + 0 +
+
+ Burst Frames (<10ms): + 0 +
+
+ Avg Frame Interval: + 0ms +
+
+ Min Frame Interval: + 0ms +
+
+ Max Frame Interval: + 0ms +
+
+ Expected FPS (30fps): + ~33.3ms +
+
+ Current Status: + Ready +
+
+
+ +
+

πŸ“Š Frame Interval Timeline (first 100 frames)

+ +
+ +
+

πŸ“ Frame Rendering Log (first 20 frames)

+
+
+
+ + + + diff --git a/samples/index.html b/samples/index.html index 295e580..8519051 100644 --- a/samples/index.html +++ b/samples/index.html @@ -82,6 +82,12 @@

MoQ Player - Samples

interactions.

+
  • + Frame Burst Test +

    + Diagnostic test to measure and demonstrate the frame burst problem at playback start. +

    +