diff --git a/lib/playback/backend.ts b/lib/playback/backend.ts index f0ad3cb..223125e 100644 --- a/lib/playback/backend.ts +++ b/lib/playback/backend.ts @@ -114,7 +114,7 @@ export default class Backend { private on(e: MessageEvent) { const msg = e.data - if (msg === "waitingforkeyframe") { + if (msg.type === "waitingforkeyframe") { this.#eventTarget.dispatchEvent(new Event("waitingforkeyframe")) } } diff --git a/lib/playback/worker/video.ts b/lib/playback/worker/video.ts index f433a57..b9846a0 100644 --- a/lib/playback/worker/video.ts +++ b/lib/playback/worker/video.ts @@ -18,18 +18,32 @@ interface DecoderConfig { optimizeForLatency?: boolean } +// Wrapper for VideoFrame with original timestamp +interface FrameWithTimestamp { + frame: VideoFrame + originalTimestamp: number +} + export class Renderer { #canvas: OffscreenCanvas #timeline: Component #decoder!: VideoDecoder - #queue: TransformStream + #queue: TransformStream #decoderConfig?: DecoderConfig #waitingForKeyframe: boolean = true #paused: boolean #hasSentWaitingForKeyFrameEvent: boolean = false + // Frame timing for proper playback rate + #playbackStartTime: number | null = null + #firstFrameTimestamp: number | null = null + + // Map to store original timestamps for frames (with memory leak protection) + #frameTimestamps: Map = new Map() + #MAX_TIMESTAMP_MAP_SIZE = 100 // Prevent memory leaks + constructor(config: Message.ConfigVideo, timeline: Component) { this.#canvas = config.canvas this.#timeline = timeline @@ -49,20 +63,57 @@ export class Renderer { console.error(err) }) this.#waitingForKeyframe = true + // Reset timing on pause so next play starts fresh + this.#playbackStartTime = null + this.#firstFrameTimestamp = null } play() { this.#paused = false + // Reset timing on play to start fresh + this.#playbackStartTime = null + this.#firstFrameTimestamp = null } async #run() { const reader = this.#timeline.frames.pipeThrough(this.#queue).getReader() + for (;;) { - const { value: frame, done } = await reader.read() + const { value: frameWithTimestamp, done } = await reader.read() if (this.#paused) continue if (done) break + const frame = frameWithTimestamp.frame + const frameTimestampMs = frameWithTimestamp.originalTimestamp + + // Initialize timing on first frame + if (this.#firstFrameTimestamp === null || this.#playbackStartTime === null) { + this.#firstFrameTimestamp = frameTimestampMs + this.#playbackStartTime = performance.now() + } + + // Calculate when this frame should be displayed + const frameOffsetMs = frameTimestampMs - this.#firstFrameTimestamp + const targetDisplayTime = this.#playbackStartTime + frameOffsetMs + + // Use requestAnimationFrame with timestamp for smoother playback self.requestAnimationFrame(() => { + // Check if still not paused (could have paused during wait) + if (this.#paused) { + frame.close() + return + } + + // Check if we should skip this frame (too late) + const now = performance.now() + const lateness = now - targetDisplayTime + + // If we're more than 50ms late, skip this frame + if (lateness > 50) { + frame.close() + return + } + this.#canvas.width = frame.displayWidth this.#canvas.height = frame.displayHeight @@ -75,10 +126,26 @@ export class Renderer { } } - #start(controller: TransformStreamDefaultController) { + #start(controller: TransformStreamDefaultController) { this.#decoder = new VideoDecoder({ output: (frame: VideoFrame) => { - controller.enqueue(frame) + // Retrieve the original timestamp from our map + const originalTimestamp = this.#frameTimestamps.get(frame.timestamp) + if (originalTimestamp !== undefined) { + // Wrap frame with original timestamp + controller.enqueue({ + frame: frame, + originalTimestamp: originalTimestamp, + }) + // Clean up the map entry + this.#frameTimestamps.delete(frame.timestamp) + } else { + // Fallback: use the frame's timestamp converted to milliseconds + controller.enqueue({ + frame: frame, + originalTimestamp: frame.timestamp / 1000, + }) + } }, error: console.error, }) @@ -140,7 +207,7 @@ export class Renderer { if (this.#waitingForKeyframe && !frame.sample.is_sync) { console.warn("Skipping non-keyframe until a keyframe is found.") if (!this.#hasSentWaitingForKeyFrameEvent) { - self.postMessage("waitingforkeyframe") + self.postMessage({ type: "waitingforkeyframe" }) this.#hasSentWaitingForKeyFrameEvent = true } return @@ -152,10 +219,26 @@ export class Renderer { this.#hasSentWaitingForKeyFrameEvent = false } + // Calculate timestamp in seconds for the chunk (standard WebCodecs unit) + const timestampSeconds = frame.sample.dts / frame.track.timescale + + // Store the original timestamp (in milliseconds) so we can retrieve it later + const timestampMs = timestampSeconds * 1000 + + // Prevent memory leak: if map gets too large, clear oldest entries + if (this.#frameTimestamps.size >= this.#MAX_TIMESTAMP_MAP_SIZE) { + const firstKey = this.#frameTimestamps.keys().next().value + if (firstKey !== undefined) { + this.#frameTimestamps.delete(firstKey) + } + } + + this.#frameTimestamps.set(timestampSeconds, timestampMs) + const chunk = new EncodedVideoChunk({ type: frame.sample.is_sync ? "key" : "delta", data: frame.sample.data, - timestamp: frame.sample.dts / frame.track.timescale, + timestamp: timestampSeconds, }) this.#decoder.decode(chunk) diff --git a/samples/frame-burst-test/frame-burst-test.html b/samples/frame-burst-test/frame-burst-test.html new file mode 100644 index 0000000..43bd804 --- /dev/null +++ b/samples/frame-burst-test/frame-burst-test.html @@ -0,0 +1,550 @@ + + + + + MoQ Player - Frame Burst Test + + + + +
+

πŸ“Š MoQ Player - Frame Burst Test

+ +
+

Test Objective

+

This test demonstrates and measures the "frame burst" problem at playback start.

+

Expected behavior: Frames should be rendered at a consistent rate (e.g., ~33ms for 30fps).

+

Actual behavior (bug): Initial frames are rendered rapidly in a burst (<5ms intervals), then playback normalizes.

+

Test criteria: FAIL if more than 5 frames are rendered with <10ms intervals at the start.

+
+ + + +
+ + +
+ +
+
+ +
+
+

πŸ“ˆ Real-time Metrics

+
+ Total Frames: + 0 +
+
+ Burst Frames (<10ms): + 0 +
+
+ Avg Frame Interval: + 0ms +
+
+ Min Frame Interval: + 0ms +
+
+ Max Frame Interval: + 0ms +
+
+ Expected FPS (30fps): + ~33.3ms +
+
+ Current Status: + Ready +
+
+
+ +
+

πŸ“Š Frame Interval Timeline (first 100 frames)

+ +
+ +
+

πŸ“ Frame Rendering Log (first 20 frames)

+
+
+
+ + + + diff --git a/samples/index.html b/samples/index.html index 295e580..8519051 100644 --- a/samples/index.html +++ b/samples/index.html @@ -82,6 +82,12 @@

MoQ Player - Samples

interactions.

+
  • + Frame Burst Test +

    + Diagnostic test to measure and demonstrate the frame burst problem at playback start. +

    +