From 4c66b2b8be484ae1e230c1ce5a8dbd5d17565fef Mon Sep 17 00:00:00 2001 From: agonchar Date: Fri, 7 Nov 2025 23:57:11 +1000 Subject: [PATCH 01/67] Skip real audio frames replaced by silence. Fix latest video and audio current ts assignment. --- src/media/buffers/shared-audio-buffer.js | 2 +- src/media/buffers/writable-audio-buffer.js | 5 +-- src/media/decoders/flow-audio.js | 5 +-- src/media/decoders/flow-video.js | 5 +-- src/media/processors/audio-gaps-processor.js | 36 +++++++++++++++++--- src/nimio.js | 1 + tests/shared-audio-buffer.test.js | 4 +-- 7 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/media/buffers/shared-audio-buffer.js b/src/media/buffers/shared-audio-buffer.js index 0a0dc4a..c0ad496 100644 --- a/src/media/buffers/shared-audio-buffer.js +++ b/src/media/buffers/shared-audio-buffer.js @@ -96,7 +96,7 @@ export class SharedAudioBuffer { } } - getLastTimestampUs() { + get lastFrameTs() { let lastIdx = this.getWriteIdx() - 1; if (lastIdx < 0) lastIdx += this.capacity; return this.timestamps[lastIdx] || 0; diff --git a/src/media/buffers/writable-audio-buffer.js b/src/media/buffers/writable-audio-buffer.js index 0cd2dfc..3b631d8 100644 --- a/src/media/buffers/writable-audio-buffer.js +++ b/src/media/buffers/writable-audio-buffer.js @@ -27,7 +27,8 @@ export class WritableAudioBuffer extends SharedAudioBuffer { } for (let i = 0; i < this._preprocessors.length; i++) { - this._preprocessors[i].process(audioFrame, this); + let pRes = this._preprocessors[i].process(audioFrame, this); + if (!pRes) return; } const writeIdx = this.getWriteIdx(); @@ -63,7 +64,7 @@ export class WritableAudioBuffer extends SharedAudioBuffer { } absorb(frameBuffer) { - let lastTs = this.getLastTimestampUs(); + let lastTs = this.lastFrameTs(); frameBuffer.forEach((frame) => { if (frame.decTimestamp > lastTs) { this.pushFrame(frame); diff --git a/src/media/decoders/flow-audio.js b/src/media/decoders/flow-audio.js index 475c048..3cdca9e 100644 --- a/src/media/decoders/flow-audio.js +++ b/src/media/decoders/flow-audio.js @@ -13,8 +13,9 @@ export class DecoderFlowAudio extends DecoderFlow { } async _handleDecoderOutput(frame, data) { - await this._handleDecodedFrame(frame); - this._state.setAudioLatestTsUs(frame.decTimestamp); + if (await this._handleDecodedFrame(frame)) { + this._state.setAudioLatestTsUs(this._buffer.lastFrameTs); + } this._state.setAudioDecoderQueue(data.decoderQueue); } } diff --git a/src/media/decoders/flow-video.js b/src/media/decoders/flow-video.js index 975d8b0..9f40b5e 100644 --- a/src/media/decoders/flow-video.js +++ b/src/media/decoders/flow-video.js @@ -11,8 +11,9 @@ export class DecoderFlowVideo extends DecoderFlow { } async _handleDecoderOutput(frame, data) { - await this._handleDecodedFrame(frame); - this._state.setVideoLatestTsUs(frame.timestamp); + if (await this._handleDecodedFrame(frame)) { + this._state.setVideoLatestTsUs(this._buffer.lastFrameTs); + } this._state.setVideoDecoderQueue(data.decoderQueue); this._state.setVideoDecoderLatency(data.decoderLatency); } diff --git a/src/media/processors/audio-gaps-processor.js b/src/media/processors/audio-gaps-processor.js index 987f151..1719b4b 100644 --- a/src/media/processors/audio-gaps-processor.js +++ b/src/media/processors/audio-gaps-processor.js @@ -1,19 +1,26 @@ export class AudioGapsProcessor { - constructor(sampleCount, sampleRate) { + constructor(sampleCount, sampleRate, logger) { this._frameLenUs = (1e6 * sampleCount) / sampleRate; this._audioTsShift = 0; + this._logger = logger; } process(frame) { - const tsdiff = frame.rawTimestamp - frame.decTimestamp - this._audioTsShift; - if (tsdiff >= 2 * this._frameLenUs && tsdiff < 1e6) { - const fillCnt = (tsdiff / this._frameLenUs) >>> 0; + let frameEffTs = frame.decTimestamp + this._audioTsShift; + if (this._checkIgnoreRealSamples(frameEffTs)) return false; + + const tsDiff = frame.rawTimestamp - frameEffTs; + if (tsDiff >= 2 * this._frameLenUs && tsDiff < 1e6) { + const fillCnt = (tsDiff / this._frameLenUs) >>> 0; for (let i = 0; i < fillCnt; i++) { - this._bufferIface.pushSilence(frame.decTimestamp + this._audioTsShift); + let silenceTs = frame.decTimestamp + this._audioTsShift; + this._bufferIface.pushSilence(silenceTs); this._audioTsShift += this._frameLenUs; + this._lastSilenceTs = silenceTs; } } frame.decTimestamp += this._audioTsShift; + return true; } setBufferIface(iface) { @@ -24,4 +31,23 @@ export class AudioGapsProcessor { this._audioTsShift = 0; this._bufferIface = null; } + + _checkIgnoreRealSamples(ts) { + if (this._lastSilenceTs > 0) { + const tsDiff = this._lastSilenceTs - ts; + if (tsDiff >= 0) { + if (tsDiff < 10_000_000) { + this._logger.debug( + "Ignore real audio frames after mute", + ts, + this._lastSilenceTs + ); + return true; + } + } else { + this._lastSilenceTs = 0; + } + } + return false; + } } diff --git a/src/nimio.js b/src/nimio.js index 2ec4b9b..e8ea80a 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -426,6 +426,7 @@ export default class Nimio { new AudioGapsProcessor( this._audioConfig.sampleCount, this._audioConfig.sampleRate, + this._logger, ), ); diff --git a/tests/shared-audio-buffer.test.js b/tests/shared-audio-buffer.test.js index 42aab94..9f30ad4 100644 --- a/tests/shared-audio-buffer.test.js +++ b/tests/shared-audio-buffer.test.js @@ -60,12 +60,12 @@ describe("SharedAudioBuffer", () => { it("returns correct last timestamp", () => { sab.setWriteIdx(1); sab.timestamps[0] = 123456; - expect(sab.getLastTimestampUs()).toBe(123456); + expect(sab.lastFrameTs).toBe(123456); }); it("returns zero if no timestamps have been written", () => { sab.setWriteIdx(0); - expect(sab.getLastTimestampUs()).toBe(0); + expect(sab.lastFrameTs).toBe(0); }); it("iterates over frames in forEach", () => { From 5d87ade40e8c79cba87f208acf5a88c0e1714c5f Mon Sep 17 00:00:00 2001 From: agonchar Date: Tue, 11 Nov 2025 16:24:50 +1000 Subject: [PATCH 02/67] Add timestamp correction --- src/media/decoders/decoder-audio.js | 4 +- src/media/decoders/decoder-video.js | 10 +-- src/media/decoders/flow.js | 12 ++- src/media/decoders/timestamp-manager.js | 115 ++++++++++++++++++++++++ src/nimio-transport.js | 4 +- src/sldp/agent.js | 66 +++++++------- 6 files changed, 164 insertions(+), 47 deletions(-) create mode 100644 src/media/decoders/timestamp-manager.js diff --git a/src/media/decoders/decoder-audio.js b/src/media/decoders/decoder-audio.js index 43758a6..634607f 100644 --- a/src/media/decoders/decoder-audio.js +++ b/src/media/decoders/decoder-audio.js @@ -93,9 +93,9 @@ self.addEventListener("message", async function (e) { frameWithHeader.byteLength, ); - timestampBuffer.push(e.data.timestamp); + timestampBuffer.push(e.data.pts); const chunkData = { - timestamp: e.data.timestamp, + timestamp: e.data.pts, type: "key", data: frame, }; diff --git a/src/media/decoders/decoder-video.js b/src/media/decoders/decoder-video.js index be4450c..dda0343 100644 --- a/src/media/decoders/decoder-video.js +++ b/src/media/decoders/decoder-video.js @@ -35,10 +35,10 @@ function handleDecoderError(error) { self.postMessage({ type: "decoderError", kind: "video" }); } -function pushChunk(data, ts) { +function pushChunk(data, time) { const encodedChunk = new EncodedVideoChunk(data); videoDecoder.decode(encodedChunk); - decodeTimings.set(encodedChunk.timestamp, ts); + decodeTimings.set(encodedChunk.timestamp, time); } function shutdownDecoder() { @@ -104,14 +104,14 @@ self.addEventListener("message", async function (e) { ); const chunkData = { - timestamp: e.data.timestamp, + timestamp: e.data.pts, type: e.data.chunkType, data: frame, }; if (!support || !support.supported) { // Buffer the chunk until the decoder is ready buffered.push({ - ts: performance.now(), + time: performance.now(), chunk: chunkData, }); return; @@ -120,7 +120,7 @@ self.addEventListener("message", async function (e) { if (buffered.length > 0) { // Process buffered chunks before the new one for (let i = 0; i < buffered.length; i++) { - pushChunk(buffered[i].chunk, buffered[i].ts); + pushChunk(buffered[i].chunk, buffered[i].time); } buffered.length = 0; } diff --git a/src/media/decoders/flow.js b/src/media/decoders/flow.js index 52b09de..6b7b4f0 100644 --- a/src/media/decoders/flow.js +++ b/src/media/decoders/flow.js @@ -62,14 +62,12 @@ export class DecoderFlow { let srcFirstTsUs = this._switchPeerFlow.firstSwitchTsUs; if ( srcFirstTsUs !== null && - Math.abs(data.timestamp - srcFirstTsUs) < SWITCH_THRESHOLD_US && - data.timestamp >= srcFirstTsUs + Math.abs(data.pts - srcFirstTsUs) < SWITCH_THRESHOLD_US && + data.pts >= srcFirstTsUs ) { // Source flow already has a frame with this timestamp, cancel input - this._logger.debug( - `Cancel input for dst from pushChunk ${data.timestamp}`, - ); - this._updateSwitchTimestamps(data.timestamp); + this._logger.debug(`Cancel input for dst from pushChunk ${data.pts}`); + this._updateSwitchTimestamps(data.pts); this._cancelInput(); return false; } @@ -79,7 +77,7 @@ export class DecoderFlow { this._decoder.postMessage( { type: "chunk", - timestamp: data.timestamp, + pts: data.pts, chunkType: data.chunkType, frameWithHeader: data.frameWithHeader, framePos: data.framePos, diff --git a/src/media/decoders/timestamp-manager.js b/src/media/decoders/timestamp-manager.js new file mode 100644 index 0000000..3aa3166 --- /dev/null +++ b/src/media/decoders/timestamp-manager.js @@ -0,0 +1,115 @@ +import { LoggersFactory } from "@/shared/logger"; + +export class TimestampManager { + constructor(instName, opts = {}) { + this._isVideo = type === "video"; + this._logger = LoggersFactory.create(instName, "TimestampManager"); + + this._dropZeroDurationFrames = !!opts.dropZeroDurationFrames; + this._adjustZeroDistDts = !!opts.adjustZeroDistDts; + this.reset(); + } + + reset() { + this._dtsDistCompensation = 0; + this._lastChunkDuration = 0; + this._lastChunk = null; + } + + validateChunk(data) { + let dts = data.pts - data.offset; + if (!this._lastChunk) { + this._setLastChunk(dts, dts, data.offset); + return true; + } + + let curChunk = this._lastChunk; + let dtsDiff = dts - curChunk.rawDts; + if (this._isVideo) { + if (dtsDiff === 0) { + if (curChunk.offset === data.offset - this._dtsDistCompensation) { + // same DTS and same offset + if (this._dropZeroDurationFrames) { + this._logger.debug( + `Drop zero duration frame ts = ${dts}, offset = ${data.offset}` + ); + return false; + } + + // TODO: how to handle this properly? + } + + if (this._adjustZeroDistDts) { + // same DTS but different offset, adjust DTS but keep PTS + if (data.offset > this._dtsDistCompensation) { + this._dtsDistCompensation += 1; + dtsDiff = 1; + data.offset -= this._dtsDistCompensation; + Logger.debug( + `Fix zero distance DTS. Total DTS compensation = ${this._dtsDistCompensation}. Offset = ${data.offset}` + ); + } else { + dtsDiff = this._revertDtsDistCompensation(); + } + } + } else if (this._dtsDistCompensation > 0 && dtsDiff > 0) { + // DTS compensation has been already applied to previous frames' DTS, so now + // it should be subtracted from current dtsDiff + let repay = Math.min(this._dtsDistCompensation, dtsDiff - 1); + dtsDiff -= repay; + this._dtsDistCompensation -= repay; + Logger.debug( + `Complete DTS compensation (${this._dtsDistCompensation}). Ts = ${dts}, offset = ${data.offset}, dtsDiff = ${dtsDiff}`, repay + ); + + if (this._dtsDistCompensation > 0) { + // can't compensate the rest of the DTS distance + dtsDiff += this._revertDtsDistCompensation(); + } + } + } + + if( this._hasDiscontinuity(dtsDiff) ) { + // Logger.debug(`Incorrect DTS difference (${dtsDiff}) between previous (ts: ${curChunk.rawDts}, offset: ${curChunk.offset}, sap: ${curChunk.sap}) and current frame (ts: ${ts}, offset: ${offset}, sap: ${isSAP})`); + dtsDiff = this._lastChunkDuration; + this._dtsDistCompensation = 0; + } + + let rawDts = dts; + dts = curChunk.dts + dtsDiff; + if( + dtsDiff > 0 && this._dtsDistCompensation === 0 || + dtsDiff > 1 + ) { + this._lastChunkDuration = dtsDiff; + } + data.pts = dts + data.offset; + + this._setLastChunk(dts, rawDts, data.offset); + return res; + } + + _revertDtsDistCompensation () { + // Can't compensate DTS distance by offset, so the only way left is to treat + // it as discontinuity. We already introduced ts shift by the compensation, + // so we should replace it with a multiple of regular dts distance. + let frameCnt = Math.floor(this._dtsDistCompensation / this._lastChunkDuration) + 1; + let result = frameCnt * this._lastChunkDuration - this._dtsDistCompensation; + this._dtsDistCompensation = 0; + + Logger.debug( + `Rollback DTS distance compensation. Frame count = ${frameCnt}, result = ${result}`, + this._dtsDistCompensation, lastDtsDist + ); + + return result; + } + + _hasDiscontinuity(tsDiff) { + return (tsDiff < 0) || (tsDiff > 10_000_000); + } + + _setLastChunk(dts, rawDts, offset) { + this._lastChunk = { dts, rawDts, offset }; + } +} diff --git a/src/nimio-transport.js b/src/nimio-transport.js index f03af67..ee991a7 100644 --- a/src/nimio-transport.js +++ b/src/nimio-transport.js @@ -138,7 +138,7 @@ export const NimioTransport = { this._metricsManager.reportBandwidth( data.trackId, data.frameWithHeader.byteLength, - data.timestamp, + data.pts, ); if (flow.processChunk(data)) return; @@ -146,7 +146,7 @@ export const NimioTransport = { if (this._isNextRenditionTrack(data.trackId)) { this._nextRenditionData.decoderFlow.processChunk(data); } else if (this._abrController?.isProbing(data.trackId)) { - this._abrController.handleChunkTs(data.timestamp); + this._abrController.handleChunkTs(data.pts); } }, diff --git a/src/sldp/agent.js b/src/sldp/agent.js index 241cc88..2b8e9d1 100644 --- a/src/sldp/agent.js +++ b/src/sldp/agent.js @@ -17,21 +17,21 @@ export class SLDPAgent { } processFrame(data) { - let frameWithHeader = new Uint8Array(data); - let trackId = frameWithHeader[0]; - let frameType = frameWithHeader[1]; + let frameWthHdr = new Uint8Array(data); + let trackId = frameWthHdr[0]; + let frameType = frameWthHdr[1]; let showTime = 0; - let frameSize = frameWithHeader.byteLength; + let frameSize = frameWthHdr.byteLength; - let dataPos = 2; + let dtPos = 2; let timestamp; if (!IS_SEQUENCE_HEADER[frameType]) { - timestamp = ByteReader.readUint(frameWithHeader, dataPos, 8); - dataPos += 8; + timestamp = ByteReader.readUint(frameWthHdr, dtPos, 8); + dtPos += 8; if (this._steady) { - showTime = ByteReader.readUint(frameWithHeader, dataPos, 8); - dataPos += 8; + showTime = ByteReader.readUint(frameWthHdr, dtPos, 8); + dtPos += 8; } } let timescale = this._timescale[trackId]; @@ -42,8 +42,7 @@ export class SLDPAgent { return; } - let tsSec; - let tsUs; + let ptsSec, ptsUs; let isKey = false; switch (frameType) { case WEB.AAC_SEQUENCE_HEADER: @@ -52,7 +51,7 @@ export class SLDPAgent { case WEB.AV1_SEQUENCE_HEADER: this._sendCodecData( trackId, - frameWithHeader.subarray(dataPos, frameSize), + frameWthHdr.subarray(dtPos, frameSize), frameType === WEB.AAC_SEQUENCE_HEADER ? "audio" : "video", frameType, ); @@ -60,14 +59,14 @@ export class SLDPAgent { case WEB.MP3: case WEB.OPUS_FRAME: if (!this._codecDataStatus[trackId]) { - let codecData = frameWithHeader.subarray(dataPos, dataPos + 4); + let codecData = frameWthHdr.subarray(dtPos, dtPos + 4); this._codecDataStatus[trackId] = true; this._sendCodecData(trackId, codecData, "audio", frameType); } case WEB.AAC_FRAME: - tsSec = timestamp / (timescale / 1000); - tsUs = Math.round(1000 * tsSec); - this._sendAudioChunk(frameWithHeader, tsUs, dataPos, showTime); + ptsSec = timestamp / (timescale / 1000); + ptsUs = Math.round(1000 * ptsSec); + this._sendAudioChunk(frameWthHdr, ptsUs, dtPos, showTime); break; case WEB.AVC_KEY_FRAME: case WEB.HEVC_KEY_FRAME: @@ -78,14 +77,17 @@ export class SLDPAgent { case WEB.AV1_FRAME: let compositionOffset = 0; if (frameType !== WEB.AV1_KEY_FRAME && frameType !== WEB.AV1_FRAME) { - compositionOffset = ByteReader.readUint(frameWithHeader, dataPos, 4); - dataPos += 4; + compositionOffset = ByteReader.readUint(frameWthHdr, dtPos, 4); + dtPos += 4; } - tsSec = (timestamp + compositionOffset) / (timescale / 1000); - tsUs = Math.round(1000 * tsSec); - // console.debug(`V frame uts: ${tsUs}, pts: ${timestamp + compositionOffset}, dts: ${timestamp}, off: ${compositionOffset}`); - this._sendVideoChunk(frameWithHeader, tsUs, isKey, dataPos, showTime); + ptsSec = (timestamp + compositionOffset) / (timescale / 1000); + ptsUs = Math.round(1000 * ptsSec); + let dtsUs = 1000 * timestamp / (timescale / 1000); + let offUs = Math.round(1000 * ptsSec - dtsUs); + + // console.debug(`V frame uts: ${ptsUs}, pts: ${timestamp + compositionOffset}, dts: ${timestamp}, off: ${compositionOffset}`); + this._sendVideoChunk(frameWthHdr, ptsUs, offUs, isKey, dtPos, showTime); break; case WEB.VP8_KEY_FRAME: case WEB.VP9_KEY_FRAME: @@ -96,9 +98,9 @@ export class SLDPAgent { isKey = true; case WEB.VP8_FRAME: case WEB.VP9_FRAME: - tsSec = timestamp / (timescale / 1000); - tsUs = Math.round(1000 * tsSec); - this._sendVideoChunk(frameWithHeader, tsUs, isKey, dataPos, showTime); + ptsSec = timestamp / (timescale / 1000); + ptsUs = Math.round(1000 * ptsSec); + this._sendVideoChunk(frameWthHdr, ptsUs, 0, isKey, dtPos, showTime); break; default: break; @@ -162,13 +164,14 @@ export class SLDPAgent { }); } - _sendVideoChunk(frameWithHeader, tsUs, isKey, dataPos, showTime) { + _sendVideoChunk(frameWithHeader, ptsUs, offUs, isKey, dtPos, showTime) { let payload = { trackId: frameWithHeader[0], - timestamp: tsUs, + pts: ptsUs, + offset: offUs, chunkType: isKey ? "key" : "delta", frameWithHeader: frameWithHeader.buffer, - framePos: dataPos, + framePos: dtPos, }; if (showTime > 0) { payload.showTime = showTime; @@ -183,12 +186,13 @@ export class SLDPAgent { ); } - _sendAudioChunk(frameWithHeader, tsUs, dataPos, showTime) { + _sendAudioChunk(frameWithHeader, ptsUs, dtPos, showTime) { let payload = { trackId: frameWithHeader[0], - timestamp: tsUs, + pts: ptsUs, + offset: 0, frameWithHeader: frameWithHeader.buffer, - framePos: dataPos, + framePos: dtPos, }; if (showTime > 0) { payload.showTime = showTime; From f03b6926d54c46b03dd5feebf03881c1e2db4e97 Mon Sep 17 00:00:00 2001 From: agonchar Date: Wed, 12 Nov 2025 14:58:12 +1000 Subject: [PATCH 03/67] Fix timestamp adjustment --- src/media/decoders/timestamp-manager.js | 49 ++++++++++++------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/media/decoders/timestamp-manager.js b/src/media/decoders/timestamp-manager.js index 3aa3166..607c44a 100644 --- a/src/media/decoders/timestamp-manager.js +++ b/src/media/decoders/timestamp-manager.js @@ -6,7 +6,6 @@ export class TimestampManager { this._logger = LoggersFactory.create(instName, "TimestampManager"); this._dropZeroDurationFrames = !!opts.dropZeroDurationFrames; - this._adjustZeroDistDts = !!opts.adjustZeroDistDts; this.reset(); } @@ -27,30 +26,29 @@ export class TimestampManager { let dtsDiff = dts - curChunk.rawDts; if (this._isVideo) { if (dtsDiff === 0) { - if (curChunk.offset === data.offset - this._dtsDistCompensation) { - // same DTS and same offset - if (this._dropZeroDurationFrames) { - this._logger.debug( - `Drop zero duration frame ts = ${dts}, offset = ${data.offset}` - ); - return false; - } - - // TODO: how to handle this properly? + const sameOffset = + curChunk.offset === data.offset - this._dtsDistCompensation; + if (sameOffset && this._dropZeroDurationFrames) { + this._logger.debug( + `Drop zero duration frame ts = ${dts}, offset = ${data.offset}` + ); + return false; } - if (this._adjustZeroDistDts) { - // same DTS but different offset, adjust DTS but keep PTS - if (data.offset > this._dtsDistCompensation) { - this._dtsDistCompensation += 1; - dtsDiff = 1; - data.offset -= this._dtsDistCompensation; - Logger.debug( - `Fix zero distance DTS. Total DTS compensation = ${this._dtsDistCompensation}. Offset = ${data.offset}` - ); - } else { - dtsDiff = this._revertDtsDistCompensation(); - } + // same DTS but different offset, adjust DTS but keep PTS + if (!sameOffset && data.offset > this._dtsDistCompensation) { + // TODO: deside on how to updates the resulting PTS, because requestAnimationFrame + // fires according to the screen update frequency, which is maximum 120-144Hz + // for now. So it make sense to update the PTS in the resulting video buffer to make + // sure all frames can be displayed. + this._dtsDistCompensation += 1; + dtsDiff = 1; + data.offset -= this._dtsDistCompensation; + Logger.debug( + `Fix zero distance DTS. Total DTS compensation = ${this._dtsDistCompensation}. Offset = ${data.offset}` + ); + } else { + dtsDiff = this._revertDtsDistCompensation(); } } else if (this._dtsDistCompensation > 0 && dtsDiff > 0) { // DTS compensation has been already applied to previous frames' DTS, so now @@ -93,13 +91,14 @@ export class TimestampManager { // Can't compensate DTS distance by offset, so the only way left is to treat // it as discontinuity. We already introduced ts shift by the compensation, // so we should replace it with a multiple of regular dts distance. - let frameCnt = Math.floor(this._dtsDistCompensation / this._lastChunkDuration) + 1; + let chunksToRepay = this._dtsDistCompensation / this._lastChunkDuration; + let frameCnt = Math.floor(chunksToRepay) + 1; let result = frameCnt * this._lastChunkDuration - this._dtsDistCompensation; this._dtsDistCompensation = 0; Logger.debug( `Rollback DTS distance compensation. Frame count = ${frameCnt}, result = ${result}`, - this._dtsDistCompensation, lastDtsDist + this._dtsDistCompensation, this._lastChunkDuration ); return result; From 069bcfa9f7fbb3c1cb671ce31b2e84bec4791b3a Mon Sep 17 00:00:00 2001 From: agonchar Date: Wed, 12 Nov 2025 23:25:41 +1000 Subject: [PATCH 04/67] Complete Timestamp manager --- src/abr/evaluator.js | 1 + src/abr/prober.js | 4 ++ src/media/decoders/flow.js | 5 +++ src/media/decoders/timestamp-manager.js | 54 ++++++++++++++++++++----- src/nimio-transport.js | 4 ++ src/nimio.js | 5 +++ src/player-config.js | 1 + 7 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/abr/evaluator.js b/src/abr/evaluator.js index 5faf6e8..017d451 100644 --- a/src/abr/evaluator.js +++ b/src/abr/evaluator.js @@ -103,6 +103,7 @@ export class AbrEvaluator { if (0 === this._runsCount && probeTime > 600) probeTime = 600; this._logger.debug(`doRun probe during ${probeTime}`); + // this._curStream.vId this._prober = new Prober(this._instName, streamToProbe.idx, probeTime); this._prober.callbacks = { onStartProbe: this._startProbeCallback, diff --git a/src/abr/prober.js b/src/abr/prober.js index b13adb8..db13669 100644 --- a/src/abr/prober.js +++ b/src/abr/prober.js @@ -1,3 +1,4 @@ +import { TimestampManager } from "@/media/decoders/timestamp-manager"; import { MetricsManager } from "@/metrics/manager"; import { LoggersFactory } from "@/shared/logger"; @@ -8,12 +9,14 @@ export class Prober { this._idx = streamIdx; this._metricsManager = MetricsManager.getInstance(instName); + this._timestampManager = TimestampManager.getInstance(instName); this._logger = LoggersFactory.create(instName, "Prober"); } destroy() { if (this._streamId) { this._metricsManager.remove(this._streamId); + this._timestampManager.removeTrack(this._streamId); } this._clearBufCheckInterval(); } @@ -30,6 +33,7 @@ export class Prober { this._logger.debug(`start: ${this._idx}, period: ${this._period}`); this._metricsManager.add(this._streamId, "probe"); + this._timestampManager.addTrack(this._streamId, "video"); } isEnabled() { diff --git a/src/media/decoders/flow.js b/src/media/decoders/flow.js index 6b7b4f0..5040ed5 100644 --- a/src/media/decoders/flow.js +++ b/src/media/decoders/flow.js @@ -1,3 +1,4 @@ +import { TimestampManager } from "./timestamp-manager"; import { MetricsManager } from "@/metrics/manager"; import { LoggersFactory } from "@/shared/logger"; @@ -20,6 +21,9 @@ export class DecoderFlow { this._metricsManager = MetricsManager.getInstance(instanceName); this._metricsManager.add(this._trackId, this._type); + this._timestampManager = TimestampManager.getInstance(instanceName); + this._timestampManager.addTrack(this._trackId, this._type); + this._decoder = new Worker(new URL(url, import.meta.url), { type: "module", }); @@ -147,6 +151,7 @@ export class DecoderFlow { this._isShuttingDown = true; this._metricsManager.remove(this._trackId); + this._timestampManager.removeTrack(this._trackId); this._decoder.postMessage({ type: "shutdown" }); } diff --git a/src/media/decoders/timestamp-manager.js b/src/media/decoders/timestamp-manager.js index 607c44a..42fe0ed 100644 --- a/src/media/decoders/timestamp-manager.js +++ b/src/media/decoders/timestamp-manager.js @@ -1,11 +1,45 @@ +import { multiInstanceService } from "@/shared/service"; import { LoggersFactory } from "@/shared/logger"; -export class TimestampManager { - constructor(instName, opts = {}) { - this._isVideo = type === "video"; - this._logger = LoggersFactory.create(instName, "TimestampManager"); +class TimestampManager { + constructor(instName) { + this._instName = instName; + this._tsValidators = new Map(); + } + + init(settings) { + this._settings = settings; + } + + addTrack(id, type) { + let tv = new TimestampValidator(this._instName, id, type, this._settings); + this._tsValidators.set(id, tv); + } + + validateChunk(id, chunk) { + let tv = this._tsValidators.get(id); + if (!tv) return false; + return tv.validateChunk(chunk); + } - this._dropZeroDurationFrames = !!opts.dropZeroDurationFrames; + resetTrack(id) { + let tv = this._tsValidators.get(id); + if (tv) tv.reset(); + } + + removeTrack(id) { + this._tsValidators.delete(id); + } + +} + +class TimestampValidator { + constructor(instName, id, type, settings) { + const name = `TS Validator [${type}][${id}]`; + this._logger = LoggersFactory.create(instName, name); + + this._isVideo = type === "video"; + this._dropZeroDurationFrames = settings.dropZeroDurationFrames; this.reset(); } @@ -75,11 +109,8 @@ export class TimestampManager { let rawDts = dts; dts = curChunk.dts + dtsDiff; - if( - dtsDiff > 0 && this._dtsDistCompensation === 0 || - dtsDiff > 1 - ) { - this._lastChunkDuration = dtsDiff; + if (dtsDiff > 1000) { + this._lastChunkDuration = dtsDiff; // 1ms at least } data.pts = dts + data.offset; @@ -112,3 +143,6 @@ export class TimestampManager { this._lastChunk = { dts, rawDts, offset }; } } + +TimestampManager = multiInstanceService(TimestampManager); +export { TimestampManager }; diff --git a/src/nimio-transport.js b/src/nimio-transport.js index ee991a7..0ccd15c 100644 --- a/src/nimio-transport.js +++ b/src/nimio-transport.js @@ -134,6 +134,10 @@ export const NimioTransport = { _processChunk(flow, data) { if (!flow) return; + if (!this._timestampManager.validateChunk(data.trackId, data)) { + this._logger.warn("Drop invalid chunk", data.trackId, data.pts); + return; + } this._metricsManager.reportBandwidth( data.trackId, diff --git a/src/nimio.js b/src/nimio.js index e8ea80a..c54da79 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -9,6 +9,7 @@ import { FrameBuffer } from "./media/buffers/frame-buffer"; import { WritableAudioBuffer } from "./media/buffers/writable-audio-buffer"; import { DecoderFlowVideo } from "./media/decoders/flow-video"; import { DecoderFlowAudio } from "./media/decoders/flow-audio"; +import { TimestampManager } from "./media/decoders/timestamp-manager"; import { createConfig } from "./player-config"; import { AudioConfig } from "./audio/config"; import { AudioGapsProcessor } from "./media/processors/audio-gaps-processor"; @@ -95,6 +96,10 @@ export default class Nimio { this._videoBuffer, ); } + this._timestampManager = TimestampManager.getInstance(this._instName); + this._timestampManager.init({ + dropZeroDurationFrames: this._config.dropZeroDurationFrames + }); this._firstFrameTsUs = 0; this._renderVideoFrame = this._renderVideoFrame.bind(this); diff --git a/src/player-config.js b/src/player-config.js index ecd0e85..fa17a32 100644 --- a/src/player-config.js +++ b/src/player-config.js @@ -21,6 +21,7 @@ const DEFAULTS = { vuMeter: null, workletLogs: false, fullscreen: false, + dropZeroDurationFrames: false, }; const REQUIRED_KEYS = ["streamUrl", "container"]; From 753c3ffb72abbae8e87a8260da5bc75eada564d9 Mon Sep 17 00:00:00 2001 From: agonchar Date: Thu, 13 Nov 2025 17:02:28 +1000 Subject: [PATCH 05/67] Integrate and finalize Timestamp manager --- src/latency-controller.js | 11 ++- src/media/decoders/flow.js | 2 + src/media/decoders/timestamp-manager.js | 76 ++++++++++++++++---- src/media/processors/audio-gaps-processor.js | 4 +- src/nimio.js | 7 +- src/sldp/agent.js | 4 +- 6 files changed, 82 insertions(+), 22 deletions(-) diff --git a/src/latency-controller.js b/src/latency-controller.js index 2fda244..c071ee6 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -123,10 +123,19 @@ export class LatencyController { _checkPending() { if (this.isUnderrun() && this._startThreshUs === 0) { + this._logger.debug( + `Buffer is underrun, set starting threshold. Available ms=${this._availableUs / 1000}`, + ); this._startThreshUs = this._startingBufferLevel(); } + let res = this.isStarting(); - if (!res) this._startThreshUs = 0; + if (!res && this._startThreshUs > 0) { + this._logger.debug( + `Buffer is full, starting. Available ms=${this._availableUs / 1000}`, + ); + this._startThreshUs = 0; + } return res; } diff --git a/src/media/decoders/flow.js b/src/media/decoders/flow.js index 5040ed5..6ab46c9 100644 --- a/src/media/decoders/flow.js +++ b/src/media/decoders/flow.js @@ -103,6 +103,8 @@ export class DecoderFlow { switchTo(flow, type = "dst") { this._switchPeerFlow = flow; if (this._startSwitch(type)) { + const peerTrackId = this._switchPeerFlow.trackId; + this._timestampManager.updateTimeBase(peerTrackId, this._trackId); this._switchPeerFlow.switchTo(this, type === "dst" ? "src" : "dst"); } } diff --git a/src/media/decoders/timestamp-manager.js b/src/media/decoders/timestamp-manager.js index 42fe0ed..db5815e 100644 --- a/src/media/decoders/timestamp-manager.js +++ b/src/media/decoders/timestamp-manager.js @@ -22,6 +22,14 @@ class TimestampManager { return tv.validateChunk(chunk); } + updateTimeBase(targetId, sourceId) { + let srcValidator = this._tsValidators.get(sourceId); + let tgtValidator = this._tsValidators.get(targetId); + if (srcValidator && tgtValidator) { + tgtValidator.timeBase = srcValidator.timeBase; + } + } + resetTrack(id) { let tv = this._tsValidators.get(id); if (tv) tv.reset(); @@ -30,14 +38,13 @@ class TimestampManager { removeTrack(id) { this._tsValidators.delete(id); } - } class TimestampValidator { constructor(instName, id, type, settings) { const name = `TS Validator [${type}][${id}]`; this._logger = LoggersFactory.create(instName, name); - + this._isVideo = type === "video"; this._dropZeroDurationFrames = settings.dropZeroDurationFrames; this.reset(); @@ -52,7 +59,14 @@ class TimestampValidator { validateChunk(data) { let dts = data.pts - data.offset; if (!this._lastChunk) { - this._setLastChunk(dts, dts, data.offset); + if (this._timeBase) { + // There is a timebase from another validator that may contain + // ts discontinuity adjustment that is needed to be applied for + // successful rendition switch + this._applyTimeBase(data); + } else { + this._setLastChunk(dts, dts, data.offset); + } return true; } @@ -64,11 +78,11 @@ class TimestampValidator { curChunk.offset === data.offset - this._dtsDistCompensation; if (sameOffset && this._dropZeroDurationFrames) { this._logger.debug( - `Drop zero duration frame ts = ${dts}, offset = ${data.offset}` + `Drop zero duration frame ts = ${dts}, offset = ${data.offset}`, ); return false; } - + // same DTS but different offset, adjust DTS but keep PTS if (!sameOffset && data.offset > this._dtsDistCompensation) { // TODO: deside on how to updates the resulting PTS, because requestAnimationFrame @@ -79,7 +93,7 @@ class TimestampValidator { dtsDiff = 1; data.offset -= this._dtsDistCompensation; Logger.debug( - `Fix zero distance DTS. Total DTS compensation = ${this._dtsDistCompensation}. Offset = ${data.offset}` + `Fix zero distance DTS. Total DTS compensation = ${this._dtsDistCompensation}. Offset = ${data.offset}`, ); } else { dtsDiff = this._revertDtsDistCompensation(); @@ -91,7 +105,8 @@ class TimestampValidator { dtsDiff -= repay; this._dtsDistCompensation -= repay; Logger.debug( - `Complete DTS compensation (${this._dtsDistCompensation}). Ts = ${dts}, offset = ${data.offset}, dtsDiff = ${dtsDiff}`, repay + `Complete DTS compensation (${this._dtsDistCompensation}). Ts = ${dts}, offset = ${data.offset}, dtsDiff = ${dtsDiff}`, + repay, ); if (this._dtsDistCompensation > 0) { @@ -101,8 +116,10 @@ class TimestampValidator { } } - if( this._hasDiscontinuity(dtsDiff) ) { - // Logger.debug(`Incorrect DTS difference (${dtsDiff}) between previous (ts: ${curChunk.rawDts}, offset: ${curChunk.offset}, sap: ${curChunk.sap}) and current frame (ts: ${ts}, offset: ${offset}, sap: ${isSAP})`); + if (this._hasDiscontinuity(dtsDiff)) { + this._logger.debug( + `Incorrect DTS difference (${dtsDiff}) between previous (ts: ${curChunk.rawDts}, offset: ${curChunk.offset}, sap: ${curChunk.sap}) and current frame (ts: ${dts}, offset: ${data.offset})`, + ); dtsDiff = this._lastChunkDuration; this._dtsDistCompensation = 0; } @@ -115,11 +132,41 @@ class TimestampValidator { data.pts = dts + data.offset; this._setLastChunk(dts, rawDts, data.offset); - return res; + return true; + } + + set timeBase(tb) { + this._timeBase = tb; + } + + get timeBase() { + if (!this._lastChunk) return null; + return { + dts: this._lastChunk.dts, + rawDts: this._lastChunk.rawDts, + }; + } + + _applyTimeBase(chunk) { + let chDts = chunk.pts - chunk.offset; + let rawDts = chDts; + if (Math.abs(chDts - this._timeBase.rawDts) < 9_000_000) { + let dtsDiff = this._timeBase.dts - this._timeBase.rawDts; + if (dtsDiff !== 0) { + let newDts = chDts + dtsDiff; + chunk.pts = newDts + chunk.offset; + this._logger.debug( + `Apply time base adjustment. DTS diff: ${dtsDiff}. Old DTS: ${chDts}, new DTS: ${newDts}, offset: ${chunk.offset}`, + ); + chDts = newDts; + } + } + this._setLastChunk(chDts, rawDts, chunk.offset); + this._timeBase = null; } - _revertDtsDistCompensation () { - // Can't compensate DTS distance by offset, so the only way left is to treat + _revertDtsDistCompensation() { + // Can't compensate DTS distance by offset, so the only way left is to treat // it as discontinuity. We already introduced ts shift by the compensation, // so we should replace it with a multiple of regular dts distance. let chunksToRepay = this._dtsDistCompensation / this._lastChunkDuration; @@ -129,14 +176,15 @@ class TimestampValidator { Logger.debug( `Rollback DTS distance compensation. Frame count = ${frameCnt}, result = ${result}`, - this._dtsDistCompensation, this._lastChunkDuration + this._dtsDistCompensation, + this._lastChunkDuration, ); return result; } _hasDiscontinuity(tsDiff) { - return (tsDiff < 0) || (tsDiff > 10_000_000); + return tsDiff < 0 || tsDiff > 10_000_000; } _setLastChunk(dts, rawDts, offset) { diff --git a/src/media/processors/audio-gaps-processor.js b/src/media/processors/audio-gaps-processor.js index 1719b4b..e3c592a 100644 --- a/src/media/processors/audio-gaps-processor.js +++ b/src/media/processors/audio-gaps-processor.js @@ -38,9 +38,7 @@ export class AudioGapsProcessor { if (tsDiff >= 0) { if (tsDiff < 10_000_000) { this._logger.debug( - "Ignore real audio frames after mute", - ts, - this._lastSilenceTs + `Ignore real audio frames after mute ts=${ts}, last silence ts=${this._lastSilenceTs}`, ); return true; } diff --git a/src/nimio.js b/src/nimio.js index c54da79..a2e1953 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -98,7 +98,7 @@ export default class Nimio { } this._timestampManager = TimestampManager.getInstance(this._instName); this._timestampManager.init({ - dropZeroDurationFrames: this._config.dropZeroDurationFrames + dropZeroDurationFrames: this._config.dropZeroDurationFrames, }); this._firstFrameTsUs = 0; @@ -279,7 +279,10 @@ export default class Nimio { if (this._latencyCtrl.isPending()) return true; const frame = this._videoBuffer.popFrameForTime(curPlayedTsUs); - if (!frame) return true; + if (!frame) { + // this._logger.debug(`No frame for ${curPlayedTsUs}, 1 frame=${this._videoBuffer.firstFrameTs}, last frame=${this._videoBuffer.lastFrameTs}, buffer length=${this._videoBuffer.length}`); + return true; + } this._ctx.drawImage( frame, diff --git a/src/sldp/agent.js b/src/sldp/agent.js index 2b8e9d1..c3db483 100644 --- a/src/sldp/agent.js +++ b/src/sldp/agent.js @@ -83,9 +83,9 @@ export class SLDPAgent { ptsSec = (timestamp + compositionOffset) / (timescale / 1000); ptsUs = Math.round(1000 * ptsSec); - let dtsUs = 1000 * timestamp / (timescale / 1000); + let dtsUs = (1000 * timestamp) / (timescale / 1000); let offUs = Math.round(1000 * ptsSec - dtsUs); - + // console.debug(`V frame uts: ${ptsUs}, pts: ${timestamp + compositionOffset}, dts: ${timestamp}, off: ${compositionOffset}`); this._sendVideoChunk(frameWthHdr, ptsUs, offUs, isKey, dtPos, showTime); break; From c31ac60c6942152a6d5c592328fd219e84fb40a0 Mon Sep 17 00:00:00 2001 From: agonchar Date: Thu, 13 Nov 2025 23:56:35 +1000 Subject: [PATCH 06/67] Add seek method draft --- src/audio/nimio-processor.js | 6 ---- src/latency-controller.js | 33 ++++++++++++++++++-- src/media/processors/audio-gaps-processor.js | 19 ----------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/audio/nimio-processor.js b/src/audio/nimio-processor.js index f9769eb..4948b24 100644 --- a/src/audio/nimio-processor.js +++ b/src/audio/nimio-processor.js @@ -95,12 +95,6 @@ class AudioNimioProcessor extends AudioWorkletProcessor { _setSpeed(speed, availableMs) { if (this._speed === speed) return; - - let tNow = currentTime; - if (speed > this._speed) { - if (this._spdSetTime > 0 && tNow - this._spdSetTime < 3) return; - } - this._spdSetTime = tNow; this._speed = speed; this._logger.debug(`speed ${speed}`, availableMs, this._targetLatencyMs); } diff --git a/src/latency-controller.js b/src/latency-controller.js index c071ee6..bdd79f9 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -16,12 +16,23 @@ export class LatencyController { this._hysteresis = this._latencyMs < 1000 ? 1.5 : 1.25; this._subHysteresis = this._latencyMs < 1000 ? 0.8 : 0.9; + let hasPerformance = typeof performance !== "undefined"; + this._timeFn = hasPerformance ? this._getPerfTime : this._getCurrentTime; + this._logger = LoggersFactory.create(instName, "Latency ctrl", params.port); this._logger.debug( `initialized: latency=${this._latencyMs}ms, start threshold=${this._startThreshUs}us, video=${this._video}, audio=${this._audio}`, ); } + _getPerfTime() { + return performance.now(); + } + + _getCurrentTime() { + return currentTime * 1000; + } + reset() { this._startTsUs = 0; this._availableUs = 0; @@ -178,12 +189,30 @@ export class LatencyController { if (this._audioAvailUs < 0) this._audioAvailUs = 0; } + + _seek(distUs) { + if (distUs <= 0) return; + + let tNow = this._timeFn(); + // if (speed > this._speed) { + if (this._lastSeekTime > 0 && tNow - this._lastSeekTime < 3000) return; + // } + this._lastSeekTime = tNow; + + this._logger.debug(`Seek forward by ${distUs}us`); + this._stateMgr.incCurrentTsSmp(this._audioConfig.tsUsToSmpCnt(distUs)); + if (this._video) this._videoAvailUs -= distUs; + if (this._audio) this._audioAvailUs -= distUs; + this._availableUs -= distUs; + } + _adjustPlaybackLatency() { let availableMs = this._meanAvailableUs.get() / 1000; if (availableMs <= this._latencyMs * this._subHysteresis) { - this._setSpeed(1.0, availableMs); + // this._setSpeed(1.0, availableMs); } else if (availableMs > this._latencyMs * this._hysteresis) { - this._setSpeed(1.1, availableMs); // speed boost + // this._setSpeed(1.1, availableMs); // speed boost + this._seek(this._availableUs - this._latencyMs * 1000); } } diff --git a/src/media/processors/audio-gaps-processor.js b/src/media/processors/audio-gaps-processor.js index e3c592a..1bcad5d 100644 --- a/src/media/processors/audio-gaps-processor.js +++ b/src/media/processors/audio-gaps-processor.js @@ -7,8 +7,6 @@ export class AudioGapsProcessor { process(frame) { let frameEffTs = frame.decTimestamp + this._audioTsShift; - if (this._checkIgnoreRealSamples(frameEffTs)) return false; - const tsDiff = frame.rawTimestamp - frameEffTs; if (tsDiff >= 2 * this._frameLenUs && tsDiff < 1e6) { const fillCnt = (tsDiff / this._frameLenUs) >>> 0; @@ -31,21 +29,4 @@ export class AudioGapsProcessor { this._audioTsShift = 0; this._bufferIface = null; } - - _checkIgnoreRealSamples(ts) { - if (this._lastSilenceTs > 0) { - const tsDiff = this._lastSilenceTs - ts; - if (tsDiff >= 0) { - if (tsDiff < 10_000_000) { - this._logger.debug( - `Ignore real audio frames after mute ts=${ts}, last silence ts=${this._lastSilenceTs}`, - ); - return true; - } - } else { - this._lastSilenceTs = 0; - } - } - return false; - } } From a907f3ab5f4a4301a5149ea283a575abf413ca23 Mon Sep 17 00:00:00 2001 From: agonchar Date: Sat, 15 Nov 2025 00:37:54 +1000 Subject: [PATCH 07/67] Improve start --- src/latency-controller.js | 21 +++++---------- src/nimio.js | 46 +++++++++++++++++++++++-------- src/shared/helpers.js | 13 +++++++++ src/shared/mean-value.js | 57 ++++++++++++++++++++++----------------- 4 files changed, 88 insertions(+), 49 deletions(-) diff --git a/src/latency-controller.js b/src/latency-controller.js index bdd79f9..714a2d8 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -1,5 +1,6 @@ import { LoggersFactory } from "@/shared/logger"; import { MeanValue } from "@/shared/mean-value"; +import { currentTimeGetterMs } from "./shared/helpers"; export class LatencyController { constructor(instName, stateMgr, audioConfig, params) { @@ -7,7 +8,7 @@ export class LatencyController { this._stateMgr = stateMgr; this._audioConfig = audioConfig; this._params = params; - this._meanAvailableUs = new MeanValue(250); + this._meanAvailableUs = new MeanValue(500); this.reset(); @@ -16,8 +17,7 @@ export class LatencyController { this._hysteresis = this._latencyMs < 1000 ? 1.5 : 1.25; this._subHysteresis = this._latencyMs < 1000 ? 0.8 : 0.9; - let hasPerformance = typeof performance !== "undefined"; - this._timeFn = hasPerformance ? this._getPerfTime : this._getCurrentTime; + this._getCurTimeMs = currentTimeGetterMs(); this._logger = LoggersFactory.create(instName, "Latency ctrl", params.port); this._logger.debug( @@ -25,14 +25,6 @@ export class LatencyController { ); } - _getPerfTime() { - return performance.now(); - } - - _getCurrentTime() { - return currentTime * 1000; - } - reset() { this._startTsUs = 0; this._availableUs = 0; @@ -193,13 +185,13 @@ export class LatencyController { _seek(distUs) { if (distUs <= 0) return; - let tNow = this._timeFn(); + let tNow = this._getCurTimeMs(); // if (speed > this._speed) { if (this._lastSeekTime > 0 && tNow - this._lastSeekTime < 3000) return; // } this._lastSeekTime = tNow; - this._logger.debug(`Seek forward by ${distUs}us`); + this._logger.debug(`Seek forward by ${distUs / 1000}ms, cur bufer ms=${this._availableUs / 1000}`); this._stateMgr.incCurrentTsSmp(this._audioConfig.tsUsToSmpCnt(distUs)); if (this._video) this._videoAvailUs -= distUs; if (this._audio) this._audioAvailUs -= distUs; @@ -212,7 +204,8 @@ export class LatencyController { // this._setSpeed(1.0, availableMs); } else if (availableMs > this._latencyMs * this._hysteresis) { // this._setSpeed(1.1, availableMs); // speed boost - this._seek(this._availableUs - this._latencyMs * 1000); + this._seek((availableMs - this._latencyMs) * 1000); + // this._meanAvailableUs.reset(); } } diff --git a/src/nimio.js b/src/nimio.js index a2e1953..20399a0 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -101,7 +101,7 @@ export default class Nimio { dropZeroDurationFrames: this._config.dropZeroDurationFrames, }); - this._firstFrameTsUs = 0; + this._resetPlaybackTimstamps(); this._renderVideoFrame = this._renderVideoFrame.bind(this); this._ctx = this._ui.canvas.getContext("2d"); @@ -210,7 +210,7 @@ export default class Nimio { this._state.setVideoLatestTsUs(0); this._state.setAudioLatestTsUs(0); this._state.resetCurrentTsSmp(); - this._firstFrameTsUs = 0; + this._resetPlaybackTimstamps(); this._ui.drawPlay(); this._ctx.clearRect(0, 0, this._ctx.canvas.width, this._ctx.canvas.height); @@ -264,7 +264,7 @@ export default class Nimio { if (this._noVideo || !this._state.isPlaying()) return true; requestAnimationFrame(this._renderVideoFrame); - if (null === this._audioWorkletReady || 0 === this._firstFrameTsUs) { + if (null === this._audioWorkletReady || 0 === this._playbackStartTsUs) { return true; } @@ -344,14 +344,19 @@ export default class Nimio { } async _onVideoStartTsNotSet(frame) { - if (this._firstFrameTsUs !== 0) return true; + if (this._playbackStartTsUs !== 0) return true; + if (this._firstVideoFrameTsUs === 0) { + this._firstVideoFrameTsUs = frame.timestamp; + if (this._firstAudioFrameTsUs > 0) { + this._setPlaybackStartTs(); + return true; + } + } if (this._noAudio || this._videoBuffer.getTimeCapacity() >= 0.5) { - this._firstFrameTsUs = frame.timestamp; - if (this._videoBuffer.length > 0) { - this._firstFrameTsUs = this._videoBuffer.firstFrameTs; + if (!this._audioContext) { + this._setPlaybackStartTs("video"); } - this._state.setPlaybackStartTsUs(this._firstFrameTsUs); if (!this._noAudio && this._audioCtxProvider.isRunning()) { await this._startNoAudioMode(); @@ -388,17 +393,36 @@ export default class Nimio { if (!this._audioContext || !this._audioNode) { this._logger.error("Audio context is not initialized. Can't play audio."); + this._audioContext = this._audioNode = null; return false; } - if (this._firstFrameTsUs === 0) { - this._firstFrameTsUs = frame.rawTimestamp; - this._state.setPlaybackStartTsUs(frame.rawTimestamp); + if (this._firstAudioFrameTsUs === 0) { + this._firstAudioFrameTsUs = frame.decTimestamp; + if (this._firstVideoFrameTsUs) { + this._setPlaybackStartTs(); + } else if (this._noVideo) { + this._setPlaybackStartTs("audio"); + } } return true; } + _setPlaybackStartTs(mode) { + this._playbackStartTsUs = + mode === "video" ? this._firstVideoFrameTsUs : + mode === "audio" ? this._firstAudioFrameTsUs : + Math.max(this._firstAudioFrameTsUs, this._firstVideoFrameTsUs); + this._logger.warn(`set playback start ts us: ${this._playbackStartTsUs}, mode: ${mode}, video: ${this._firstVideoFrameTsUs}, audio: ${this._firstAudioFrameTsUs}`); + this._state.setPlaybackStartTsUs(this._playbackStartTsUs); + } + + _resetPlaybackTimstamps() { + this._playbackStartTsUs = 0; + this._firstAudioFrameTsUs = this._firstVideoFrameTsUs = 0; + } + _onDecodingError(kind) { // TODO: show error message in UI if (kind === "video") this._setNoVideo(); diff --git a/src/shared/helpers.js b/src/shared/helpers.js index d009f70..a64e705 100644 --- a/src/shared/helpers.js +++ b/src/shared/helpers.js @@ -5,3 +5,16 @@ export function mean(arr) { }); return result === 0 ? result : result / arr.length; } + +export function currentTimeGetterMs() { + function getPerfTime() { + return performance.now(); + } + + function getCurrentTime() { + return currentTime * 1000; + } + + let hasPerformance = typeof performance !== "undefined"; + return hasPerformance ? getPerfTime : getCurrentTime; +} diff --git a/src/shared/mean-value.js b/src/shared/mean-value.js index a714148..9cd4d9f 100644 --- a/src/shared/mean-value.js +++ b/src/shared/mean-value.js @@ -1,51 +1,60 @@ +import { currentTimeGetterMs } from "./helpers"; + export class MeanValue { constructor(period = 100) { - this._count = 0; - this._sum = 0; + // this._count = 0; + // this._sum = 0; + this._alpha = 0.15; this._periodMs = period; - let hasPerformance = typeof performance !== "undefined"; - this._timeFn = hasPerformance ? this._getPerfTime : this._getCurrentTime; + this._getCurTime = currentTimeGetterMs(); } add(value) { this._checkTimer(); - this._sum += value; - this._count++; + if (value < this._min || this._min === undefined) { + this._min = value; + } + if (this._avg !== undefined) { + this._avg = this._alpha * value + (1 - this._alpha) * this._avg; + } else { + this._avg = value; + } + // this._sum += value; + // this._count++; } get() { + let prev = this._min; this._checkTimer(); - if (this._count === 0) return 0; - return this._sum / this._count; + if (this._min === undefined) this._min = prev; + // if (this._count === 0) return 0; + // return this._sum / this._count; + return this._min; + // return this._avg; } reset() { this._t1Ms = null; - this._count = 0; - this._sum = 0; + this._min = undefined; + this._avg = undefined; + // this._count = 0; + // this._sum = 0; } _checkTimer() { if (!this._t1Ms) { - this._t1Ms = this._timeFn(); + this._t1Ms = this._getCurTime(); return; } - let t2Ms = this._timeFn(); + let t2Ms = this._getCurTime(); if (t2Ms - this._t1Ms >= this._periodMs) { this._t1Ms = t2Ms; - if (this._count !== 0) { - this._sum /= this._count; - this._count = 1; - } + this._min = undefined; + // if (this._count !== 0) { + // this._sum /= this._count; + // this._count = 1; + // } } } - - _getPerfTime() { - return performance.now(); - } - - _getCurrentTime() { - return currentTime * 1000; - } } From b659346cf2b0452d4a3ff2c170b4ed1c06bb2adf Mon Sep 17 00:00:00 2001 From: agonchar Date: Tue, 18 Nov 2025 00:46:32 +1000 Subject: [PATCH 08/67] Add sliding window calculations --- src/latency-controller.js | 21 ++++++++-------- src/nimio.js | 14 +++++++---- src/player-config.js | 50 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/latency-controller.js b/src/latency-controller.js index 714a2d8..ec29235 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -181,7 +181,6 @@ export class LatencyController { if (this._audioAvailUs < 0) this._audioAvailUs = 0; } - _seek(distUs) { if (distUs <= 0) return; @@ -191,7 +190,9 @@ export class LatencyController { // } this._lastSeekTime = tNow; - this._logger.debug(`Seek forward by ${distUs / 1000}ms, cur bufer ms=${this._availableUs / 1000}`); + this._logger.debug( + `Seek forward by ${distUs / 1000}ms, cur bufer ms=${this._availableUs / 1000}`, + ); this._stateMgr.incCurrentTsSmp(this._audioConfig.tsUsToSmpCnt(distUs)); if (this._video) this._videoAvailUs -= distUs; if (this._audio) this._audioAvailUs -= distUs; @@ -199,14 +200,14 @@ export class LatencyController { } _adjustPlaybackLatency() { - let availableMs = this._meanAvailableUs.get() / 1000; - if (availableMs <= this._latencyMs * this._subHysteresis) { - // this._setSpeed(1.0, availableMs); - } else if (availableMs > this._latencyMs * this._hysteresis) { - // this._setSpeed(1.1, availableMs); // speed boost - this._seek((availableMs - this._latencyMs) * 1000); - // this._meanAvailableUs.reset(); - } + // let availableMs = this._meanAvailableUs.get() / 1000; + // if (availableMs <= this._latencyMs * this._subHysteresis) { + // // this._setSpeed(1.0, availableMs); + // } else if (availableMs > this._latencyMs * this._hysteresis) { + // // this._setSpeed(1.1, availableMs); // speed boost + // this._seek((availableMs - this._latencyMs) * 1000); + // // this._meanAvailableUs.reset(); + // } } _startingBufferLevel() { diff --git a/src/nimio.js b/src/nimio.js index 20399a0..ad61f46 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -411,10 +411,14 @@ export default class Nimio { _setPlaybackStartTs(mode) { this._playbackStartTsUs = - mode === "video" ? this._firstVideoFrameTsUs : - mode === "audio" ? this._firstAudioFrameTsUs : - Math.max(this._firstAudioFrameTsUs, this._firstVideoFrameTsUs); - this._logger.warn(`set playback start ts us: ${this._playbackStartTsUs}, mode: ${mode}, video: ${this._firstVideoFrameTsUs}, audio: ${this._firstAudioFrameTsUs}`); + mode === undefined + ? Math.max(this._firstAudioFrameTsUs, this._firstVideoFrameTsUs) + : mode === "video" + ? this._firstVideoFrameTsUs + : this._firstAudioFrameTsUs; + this._logger.warn( + `set playback start ts us: ${this._playbackStartTsUs}, mode: ${mode}, video: ${this._firstVideoFrameTsUs}, audio: ${this._firstAudioFrameTsUs}`, + ); this._state.setPlaybackStartTsUs(this._playbackStartTsUs); } @@ -586,6 +590,8 @@ export default class Nimio { this._audioConfig, { latency: this._config.latency, + tolerance: this._config.latencyTolerance, + adjustMethod: this._config.latencyAdjustMethod, video: !this._noVideo, audio: !this._noAudio, }, diff --git a/src/player-config.js b/src/player-config.js index fa17a32..1de62f4 100644 --- a/src/player-config.js +++ b/src/player-config.js @@ -6,7 +6,9 @@ const DEFAULTS = { autoplay: false, width: 476, height: 268, - latency: 200, + latency: 300, + latencyTolerance: "auto", + latencyAdjustMethod: "fast-forward", startOffset: 1000, pauseTimeout: 3000, metricsOverlay: false, @@ -25,6 +27,7 @@ const DEFAULTS = { }; const REQUIRED_KEYS = ["streamUrl", "container"]; +const MIN_LATENCY = 100; function validateRequired(cfg) { REQUIRED_KEYS.forEach((key) => { @@ -35,6 +38,50 @@ function validateRequired(cfg) { }); } +function initLatencySettings(settings, logger) { + settings.latency = parseInt(settings.latency); + if (isNaN(settings.latency) || settings.latency < MIN_LATENCY) { + let err = + settings.latency < MIN_LATENCY + ? `less than minimum ${MIN_LATENCY} ms` + : "invalid"; + let val = settings.latency < MIN_LATENCY ? MIN_LATENCY : DEFAULTS.latency; + logger.error( + `Parameter latency=${settings.latency} is ${err}. Setting to ${DEFAULTS.latency} ms`, + ); + settings.latency = val; + } + + if (settings.latencyTolerance !== "auto") { + settings.latencyTolerance = parseInt(settings.latencyTolerance); + if ( + isNaN(settings.latencyTolerance) || + settings.latencyTolerance < settings.latency + ) { + let err = + settings.latencyTolerance < settings.latency + ? `less than latency ${settings.latency} ms` + : "invalid"; + logger.error( + `Parameter latencyTolerance=${settings.latencyTolerance} is ${err}. Setting to "auto"`, + ); + settings.latencyTolerance = "auto"; + } + } + + if (settings.latencyTolerance === "auto") { + settings.latencyTolerance = settings.latency; + settings.latencyTolerance += Math.min(settings.latency, 1000); + } + + if (!["fast-forward", "seek"].includes(settings.latencyAdjustMethod)) { + logger.error( + `Parameter latencyAdjustMethod=${settings.latencyAdjustMethod} is invalid. Setting to default "fast-forward"`, + ); + settings.latencyAdjustMethod = "fast-forward"; + } +} + // TODO: make more friendly setting of initialRendition and maxRendition // Like "480p" or 480, "1280x720", "min", "max", etc. function initAbrSettings(settings, logger) { @@ -141,6 +188,7 @@ export function createConfig(overrides = {}) { target.fullBufferMs = target.latency + target.startOffset + target.pauseTimeout; + initLatencySettings(target, logger); initAbrSettings(target, logger); initVUMeterSettings(target, logger); From f1ebfbe7e52e8cf0001cbe85e35e41158a12587e Mon Sep 17 00:00:00 2001 From: agonchar Date: Tue, 18 Nov 2025 23:03:53 +1000 Subject: [PATCH 09/67] Improve latency controller --- src/latency-controller.js | 1 - src/latency/buffer-meter.js | 50 +++++++++++++++++++++ src/player-config.js | 2 +- src/shared/mean-value.js | 60 -------------------------- src/shared/ring-buffer.js | 64 ++++++++++++++------------- src/shared/ring-queue.js | 86 +++++++++++++++++++++++++++++++++++++ src/shared/sliding-min.js | 42 ++++++++++++++++++ 7 files changed, 213 insertions(+), 92 deletions(-) create mode 100644 src/latency/buffer-meter.js delete mode 100644 src/shared/mean-value.js create mode 100644 src/shared/ring-queue.js create mode 100644 src/shared/sliding-min.js diff --git a/src/latency-controller.js b/src/latency-controller.js index ec29235..8815282 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -1,5 +1,4 @@ import { LoggersFactory } from "@/shared/logger"; -import { MeanValue } from "@/shared/mean-value"; import { currentTimeGetterMs } from "./shared/helpers"; export class LatencyController { diff --git a/src/latency/buffer-meter.js b/src/latency/buffer-meter.js new file mode 100644 index 0000000..7790194 --- /dev/null +++ b/src/latency/buffer-meter.js @@ -0,0 +1,50 @@ +class LatencyBufferMeter { + constructor(instName) { + this._shortWindowMs = 300; + this._longWindowMs = 2000; + + this._shortMin = new SlidingWindowMin(instName, 96, this._shortWindowMs); + this._longMin = new SlidingWindowMin(instName, 512, this._longWindowMs); + + this._lastValue = 0; + } + + update(bufMs, nowMs) { + this._lastValue = bufMs; + + this._shortMin.push(bufMs, nowMs); + this._longMin.push(bufMs, nowMs); + this._updateEma(bufMs); + } + + get ema() { + return this.ema == null ? 0 : this.ema; + } + + get shortBuffer() { + return this._shortMin.getMin(performance.now()) / 1000; + } + + get longBuffer() { + return this._longMin.getMin(performance.now()) / 1000; + } + + get estimatedBuffer() { + const now = performance.now(); + const shortB = this._shortMin.getMin(now); + const longB = this._longMin.getMin(now); + + return 0.6 * shortB + 0.4 * longB; + } + + reset() { + this._shortMin.clear(); + this._longMin.clear(); + this._lastValue = 0; + } + + _updateEma(value) { + if (this.ema == null) this.ema = value; + else this.ema = this.emaAlpha * value + (1 - this.emaAlpha) * this.ema; + } +} \ No newline at end of file diff --git a/src/player-config.js b/src/player-config.js index 1de62f4..1f8d2b1 100644 --- a/src/player-config.js +++ b/src/player-config.js @@ -6,7 +6,7 @@ const DEFAULTS = { autoplay: false, width: 476, height: 268, - latency: 300, + latency: 200, latencyTolerance: "auto", latencyAdjustMethod: "fast-forward", startOffset: 1000, diff --git a/src/shared/mean-value.js b/src/shared/mean-value.js deleted file mode 100644 index 9cd4d9f..0000000 --- a/src/shared/mean-value.js +++ /dev/null @@ -1,60 +0,0 @@ -import { currentTimeGetterMs } from "./helpers"; - -export class MeanValue { - constructor(period = 100) { - // this._count = 0; - // this._sum = 0; - this._alpha = 0.15; - this._periodMs = period; - this._getCurTime = currentTimeGetterMs(); - } - - add(value) { - this._checkTimer(); - if (value < this._min || this._min === undefined) { - this._min = value; - } - if (this._avg !== undefined) { - this._avg = this._alpha * value + (1 - this._alpha) * this._avg; - } else { - this._avg = value; - } - // this._sum += value; - // this._count++; - } - - get() { - let prev = this._min; - this._checkTimer(); - if (this._min === undefined) this._min = prev; - // if (this._count === 0) return 0; - // return this._sum / this._count; - return this._min; - // return this._avg; - } - - reset() { - this._t1Ms = null; - this._min = undefined; - this._avg = undefined; - // this._count = 0; - // this._sum = 0; - } - - _checkTimer() { - if (!this._t1Ms) { - this._t1Ms = this._getCurTime(); - return; - } - - let t2Ms = this._getCurTime(); - if (t2Ms - this._t1Ms >= this._periodMs) { - this._t1Ms = t2Ms; - this._min = undefined; - // if (this._count !== 0) { - // this._sum /= this._count; - // this._count = 1; - // } - } - } -} diff --git a/src/shared/ring-buffer.js b/src/shared/ring-buffer.js index 2588745..56250ef 100644 --- a/src/shared/ring-buffer.js +++ b/src/shared/ring-buffer.js @@ -7,34 +7,34 @@ export class RingBuffer { throw `Invalid capacity ${capacity}`; } - this.buffer = new Array(capacity); - this.capacity = capacity; - this.head = this.tail = this.length = 0; + this._buf = new Array(capacity); + this._cap = capacity; + this._head = this._tail = this._length = 0; } isFull() { - return this.length === this.capacity; + return this._length === this._cap; } isEmpty() { - return this.length === 0; + return this._length === 0; } push(item, force = false) { if (this.isFull()) { if (!force) { - this._logger.error(`Ring buffer is full. Capacity: ${this.capacity}`); + this._logger.error(`Ring buffer is full. Capacity: ${this._cap}`); return; } - this.length--; // decrease length to allow increment below - this.head++; - if (this.head === this.capacity) this.head = 0; + this._length--; // decrease length to allow increment below + this._head++; + if (this._head === this._cap) this._head = 0; } - this.buffer[this.tail++] = item; - if (this.tail === this.capacity) this.tail = 0; - this.length++; + this._buf[this._tail++] = item; + if (this._tail === this._cap) this._tail = 0; + this._length++; } pop() { @@ -43,7 +43,7 @@ export class RingBuffer { return null; } - const item = this.buffer[this.head]; + const item = this._buf[this._head]; this.skip(); return item; @@ -56,35 +56,35 @@ export class RingBuffer { } let i = idx; - if (i < 0) i += this.length; - if (i < 0 || i >= this.length || i == undefined) { - this._logger.error("Invalid index for get", idx, this.length); + if (i < 0) i += this._length; + if (i < 0 || i >= this._length || i == undefined) { + this._logger.error("Invalid index for get", idx, this._length); return null; } - let index = this.head + i; - if (index >= this.capacity) index -= this.capacity; - return this.buffer[index]; + let index = this._head + i; + if (index >= this._cap) index -= this._cap; + return this._buf[index]; } skip() { - this.buffer[this.head] = undefined; - this.length--; - this.head++; - if (this.head === this.capacity) this.head = 0; + this._buf[this._head] = undefined; + this._length--; + this._head++; + if (this._head === this._cap) this._head = 0; } reset() { - this.head = this.tail = this.length = 0; - this.buffer.length = 0; // fast reset of the array - this.buffer.length = this.capacity; + this._head = this._tail = this._length = 0; + this._buf.length = 0; // fast reset of the array + this._buf.length = this._cap; } forEach(fn) { - let index = this.head; - for (let i = 0; i < this.length; i++) { - fn(this.buffer[index++]); - if (index >= this.capacity) index -= this.capacity; + let index = this._head; + for (let i = 0; i < this._length; i++) { + fn(this._buf[index++]); + if (index >= this._cap) index -= this._cap; } } @@ -95,4 +95,8 @@ export class RingBuffer { }); return result; } + + get length() { + return this._length; + } } diff --git a/src/shared/ring-queue.js b/src/shared/ring-queue.js new file mode 100644 index 0000000..8d54a4c --- /dev/null +++ b/src/shared/ring-queue.js @@ -0,0 +1,86 @@ +import { LoggersFactory } from "./logger"; + +export class RingQueue { + constructor(instName, capacity = 512) { + this._logger = LoggersFactory.create(instName, "RingQueue"); + if (!Number.isInteger(capacity) || capacity <= 0) { + throw `Invalid capacity ${capacity}`; + } + + this._buf = new Array(capacity); + this._cap = capacity; + this.clear(); + } + + clear() { + this._head = this._tail = this._length = 0; + } + + isFull() { + return this._length === this._cap; + } + + isEmpty() { + return this._length === 0; + } + + pushBack(v) { + this._buf[this._tail] = v; + this._tail = this._next(this._tail); + if (this._length < this._cap) { + this._length++; + } else { + this._logger.warn("pushBack() overflow"); + this._head = this._next(this._head); + } + } + + popBack() { + if (this.isEmpty()) return null; + + this._tail = this._prev(this._tail); + const v = this._buf[this._tail]; + this._length--; + return v; + } + + pushFront(v) { + this._head = this._prev(this._head); + this._buf[this._head] = v; + if (this._length < this._cap) { + this._length++; + } else { + this._logger.warn("pushFront() overflow"); + this._tail = this._prev(this._tail); + } + } + + popFront() { + if (this.isEmpty()) return null; + + const v = this._buf[this._head]; + this._head = this._next(this._head); + this._length--; + return v; + } + + front() { + return this.isEmpty() ? null : this._buf[this._head]; + } + + back() { + return this.isEmpty() ? null : this._buf[this._prev(this._tail)]; + } + + get length() { + return this._length; + } + + _next(i) { + return (i + 1) % this._cap; + } + + _prev(i) { + return (i - 1 + this._cap) % this._cap; + } +} diff --git a/src/shared/sliding-min.js b/src/shared/sliding-min.js new file mode 100644 index 0000000..1dbdc91 --- /dev/null +++ b/src/shared/sliding-min.js @@ -0,0 +1,42 @@ +class SlidingMin { + constructor(instName, windowSizeMs, capacity = 512) { + this._windowSize = windowSizeMs; + this._vals = new RingQueue(instName, capacity); + this._times = new RingQueue(instName, capacity); + } + + push(value, timeMs) { + // Remove all larger values from the tail + while (!this._vals.isEmpty() && this._vals.back() > value) { + this._vals.popBack(); + this._times.popBack(); + } + + this._vals.pushBack(value); + this._times.pushBack(timeMs); + + this._expire(timeMs); + } + + getMin(timeMs) { + this._expire(timeMs) + + if (this._vals.length === 0) return null; + return this._vals.front(); + } + + clear() { + this._times.clear(); + this._vals.clear(); + } + + _expire(timeMs) { + // Remove old items outside the window (from the head) + const cutoff = timeMs - this._windowSize; + while (!this._times.isEmpty() && this._times.front() < cutoff) { + this._times.popFront(); + this._vals.popFront(); + } + } +} + \ No newline at end of file From 1d8deaac664d55659af121e193d24078709f6627 Mon Sep 17 00:00:00 2001 From: agonchar Date: Wed, 19 Nov 2025 23:29:21 +1000 Subject: [PATCH 10/67] Add dynamic rate change for latency adjustment --- src/latency-controller.js | 77 ++++++++++++++++++++++++++++++++++--- src/latency/buffer-meter.js | 52 ++++++++++++++----------- src/shared/helpers.js | 4 ++ src/shared/ring-queue.js | 4 +- src/shared/sliding-min.js | 10 +++-- 5 files changed, 113 insertions(+), 34 deletions(-) diff --git a/src/latency-controller.js b/src/latency-controller.js index 8815282..1b7310e 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -1,5 +1,7 @@ +import { LatencyBufferMeter } from "@/latency/buffer-meter"; import { LoggersFactory } from "@/shared/logger"; -import { currentTimeGetterMs } from "./shared/helpers"; +import { currentTimeGetterMs } from "@/shared/helpers"; +import { clamp } from "@/shared/helpers"; export class LatencyController { constructor(instName, stateMgr, audioConfig, params) { @@ -7,7 +9,14 @@ export class LatencyController { this._stateMgr = stateMgr; this._audioConfig = audioConfig; this._params = params; - this._meanAvailableUs = new MeanValue(500); + + this._shortWindowMs = 300; + this._longWindowMs = 1500; + this._bufferMeter = new LatencyBufferMeter( + instName, + this._shortWindowMs, + this._longWindowMs + ); this.reset(); @@ -16,6 +25,15 @@ export class LatencyController { this._hysteresis = this._latencyMs < 1000 ? 1.5 : 1.25; this._subHysteresis = this._latencyMs < 1000 ? 0.8 : 0.9; + this._warmupMs = 3000; + this._holdMs = 500; + this._minRate = 0.95; + this._maxRate = 1.1; + this._rateK = 0.00015; // proportional gain: rate = 1 + rateK * deltaMs + + this._minRateChangeIntervalMs = 200; + this._minSeekIntervalMs = 2000; // don't seek more frequently + this._getCurTimeMs = currentTimeGetterMs(); this._logger = LoggersFactory.create(instName, "Latency ctrl", params.port); @@ -28,8 +46,9 @@ export class LatencyController { this._startTsUs = 0; this._availableUs = 0; this._prevVideoTime = 0; - this._meanAvailableUs.reset(); + this._bufferMeter.reset(); + this._startTime = 0; this._audioAvailUs = this._videoAvailUs = undefined; this._audio = !!this._params.audio; @@ -137,6 +156,9 @@ export class LatencyController { `Buffer is full, starting. Available ms=${this._availableUs / 1000}`, ); this._startThreshUs = 0; + if (this._startTime === 0) { + this._startTime = this._getCurTimeMs(); + } } return res; } @@ -158,7 +180,9 @@ export class LatencyController { this._availableUs = Math.min(this._availableUs, this._videoAvailUs); } - this._meanAvailableUs.add(this._availableUs); + if (this._startTime > 0) { + this._bufferMeter.update(this._availableUs / 1000, this._getCurTimeMs()); + } } _getCurrentTsUs() { @@ -199,14 +223,49 @@ export class LatencyController { } _adjustPlaybackLatency() { - // let availableMs = this._meanAvailableUs.get() / 1000; + if (this._startTime === 0) return; // not started yet + + // this._logger.debug(`Available ms=${this._availableUs / 1000}, ${this._bufferMeter.get()}`); + // let availableMs = this._bufferMeter.get() / 1000; // if (availableMs <= this._latencyMs * this._subHysteresis) { // // this._setSpeed(1.0, availableMs); // } else if (availableMs > this._latencyMs * this._hysteresis) { // // this._setSpeed(1.1, availableMs); // speed boost // this._seek((availableMs - this._latencyMs) * 1000); - // // this._meanAvailableUs.reset(); + // // this._bufferMeter.reset(); // } + + const now = this._getCurTimeMs(); + const age = now - this._startTime; + const useShort = (age < this._warmupMs); + + const bufMin = useShort ? this._bufferMeter.short(now) : this._bufferMeter.long(now); + const bufEma = this._bufferMeter.ema(); + + const delta = bufMin - this._latencyMs; + // const stable = Math.abs(bufMin - bufEma) < (this._latencyMs * 0.1); + + // wait for holdMs to avoid acting on single spikes + let doAdjustment = false; + if (delta > 20) { + if (!this._pendingStableSince) { + this._pendingStableSince = now; + } else if (now - this._pendingStableSince > this._holdMs) { + doAdjustment = true; + } + } else { + this._pendingStableSince = null; + } + + let rate = 1.0; + if (doAdjustment) { + if (now - this._lastActionTime < this._minRateChangeIntervalMs) { + return; + } + rate = clamp(1 + (this._rateK * delta), this._minRate, this._maxRate); + } + + this._applyPlaybackRate(rate, bufMin, now); } _startingBufferLevel() { @@ -221,4 +280,10 @@ export class LatencyController { // this._startTime += targetLatencyMs; // } // } + + _applyPlaybackRate(rate, availableMs, now) { + if (Math.abs(rate - 1.0) < 0.001) rate = 1.0; // snap + this._setSpeed(rate, availableMs); + this._lastActionTime = now; + } } diff --git a/src/latency/buffer-meter.js b/src/latency/buffer-meter.js index 7790194..2c3b892 100644 --- a/src/latency/buffer-meter.js +++ b/src/latency/buffer-meter.js @@ -1,38 +1,42 @@ -class LatencyBufferMeter { - constructor(instName) { - this._shortWindowMs = 300; - this._longWindowMs = 2000; +import { SlidingMin } from "@/shared/sliding-min"; - this._shortMin = new SlidingWindowMin(instName, 96, this._shortWindowMs); - this._longMin = new SlidingWindowMin(instName, 512, this._longWindowMs); +export class LatencyBufferMeter { + constructor(instName, shortWindowMs, longWindowMs) { + this._shortWindowMs = shortWindowMs; + this._longWindowMs = longWindowMs; + this._emaAlpha = 0.15; - this._lastValue = 0; + let fCnt = this._fpw(10 * this._shortWindowMs); // 10 times reserve just in case + this._shortMin = new SlidingMin(instName, this._shortWindowMs, fCnt); + fCnt = this._fpw(10 * this._longWindowMs); + this._longMin = new SlidingMin(instName, this._longWindowMs, fCnt); } update(bufMs, nowMs) { - this._lastValue = bufMs; - this._shortMin.push(bufMs, nowMs); this._longMin.push(bufMs, nowMs); this._updateEma(bufMs); } - get ema() { - return this.ema == null ? 0 : this.ema; + get(timeMs) { + return `shortMin=${this.short(timeMs)?.toFixed(4)}ms, longMin=${this.long(timeMs)?.toFixed(4)}ms, est=${this.estimatedBuffer(timeMs)?.toFixed(4)}ms, ema=${this.ema().toFixed(4)}ms`; + } + + short(timeMs) { + return this._shortMin.getMin(timeMs); } - get shortBuffer() { - return this._shortMin.getMin(performance.now()) / 1000; + long(timeMs) { + return this._longMin.getMin(timeMs); } - get longBuffer() { - return this._longMin.getMin(performance.now()) / 1000; + ema() { + return this._ema || 0; } - get estimatedBuffer() { - const now = performance.now(); - const shortB = this._shortMin.getMin(now); - const longB = this._longMin.getMin(now); + estimatedBuffer(timeMs) { + const shortB = this._shortMin.getMin(timeMs); + const longB = this._longMin.getMin(timeMs); return 0.6 * shortB + 0.4 * longB; } @@ -40,11 +44,15 @@ class LatencyBufferMeter { reset() { this._shortMin.clear(); this._longMin.clear(); - this._lastValue = 0; + this._ema = undefined; } _updateEma(value) { - if (this.ema == null) this.ema = value; - else this.ema = this.emaAlpha * value + (1 - this.emaAlpha) * this.ema; + if (this._ema === undefined) this._ema = value; + else this._ema = this._emaAlpha * value + (1 - this._emaAlpha) * this._ema; + } + + _fpw(sizeMs) { // max frames per window + return Math.ceil(60 * sizeMs / 1000); } } \ No newline at end of file diff --git a/src/shared/helpers.js b/src/shared/helpers.js index a64e705..1ed7eda 100644 --- a/src/shared/helpers.js +++ b/src/shared/helpers.js @@ -18,3 +18,7 @@ export function currentTimeGetterMs() { let hasPerformance = typeof performance !== "undefined"; return hasPerformance ? getPerfTime : getCurrentTime; } + +export function clamp(val, lo, hi) { + return Math.max(lo, Math.min(hi, val)); +} diff --git a/src/shared/ring-queue.js b/src/shared/ring-queue.js index 8d54a4c..654c804 100644 --- a/src/shared/ring-queue.js +++ b/src/shared/ring-queue.js @@ -64,11 +64,11 @@ export class RingQueue { return v; } - front() { + get front() { return this.isEmpty() ? null : this._buf[this._head]; } - back() { + get back() { return this.isEmpty() ? null : this._buf[this._prev(this._tail)]; } diff --git a/src/shared/sliding-min.js b/src/shared/sliding-min.js index 1dbdc91..d80203a 100644 --- a/src/shared/sliding-min.js +++ b/src/shared/sliding-min.js @@ -1,4 +1,6 @@ -class SlidingMin { +import { RingQueue } from "./ring-queue"; + +export class SlidingMin { constructor(instName, windowSizeMs, capacity = 512) { this._windowSize = windowSizeMs; this._vals = new RingQueue(instName, capacity); @@ -7,7 +9,7 @@ class SlidingMin { push(value, timeMs) { // Remove all larger values from the tail - while (!this._vals.isEmpty() && this._vals.back() > value) { + while (!this._vals.isEmpty() && this._vals.back > value) { this._vals.popBack(); this._times.popBack(); } @@ -22,7 +24,7 @@ class SlidingMin { this._expire(timeMs) if (this._vals.length === 0) return null; - return this._vals.front(); + return this._vals.front; } clear() { @@ -33,7 +35,7 @@ class SlidingMin { _expire(timeMs) { // Remove old items outside the window (from the head) const cutoff = timeMs - this._windowSize; - while (!this._times.isEmpty() && this._times.front() < cutoff) { + while (!this._times.isEmpty() && this._times.front < cutoff) { this._times.popFront(); this._vals.popFront(); } From 5b0b473e2ea878fc331a5fe396442bac8dc06232 Mon Sep 17 00:00:00 2001 From: agonchar Date: Thu, 20 Nov 2025 23:01:46 +1000 Subject: [PATCH 11/67] Adjust latency controller parameters --- src/audio/nimio-processor.js | 2 + src/latency-controller.js | 103 +++++++++++++++-------------------- src/nimio.js | 2 + 3 files changed, 49 insertions(+), 58 deletions(-) diff --git a/src/audio/nimio-processor.js b/src/audio/nimio-processor.js index 4948b24..2972462 100644 --- a/src/audio/nimio-processor.js +++ b/src/audio/nimio-processor.js @@ -34,6 +34,8 @@ class AudioNimioProcessor extends AudioWorkletProcessor { this._audioConfig, { latency: this._targetLatencyMs, + tolerance: options.processorOptions.latencyTolerance, + adjustMethod: options.processorOptions.latencyAdjustMethod, video: options.processorOptions.videoEnabled, audio: !this._idle, port: this.port, diff --git a/src/latency-controller.js b/src/latency-controller.js index 1b7310e..5113196 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -17,6 +17,7 @@ export class LatencyController { this._shortWindowMs, this._longWindowMs ); + this._adjustFn = params.adjustMethod === "seek" ? this._seek : this._zap; this.reset(); @@ -31,8 +32,9 @@ export class LatencyController { this._maxRate = 1.1; this._rateK = 0.00015; // proportional gain: rate = 1 + rateK * deltaMs - this._minRateChangeIntervalMs = 200; - this._minSeekIntervalMs = 2000; // don't seek more frequently + this._minLatencyDelta = 40; + this._minRateChangeIntervalMs = 500; + this._minSeekIntervalMs = 4000; // don't seek more frequently this._getCurTimeMs = currentTimeGetterMs(); @@ -48,7 +50,7 @@ export class LatencyController { this._prevVideoTime = 0; this._bufferMeter.reset(); - this._startTime = 0; + this._startTimeMs = 0; this._audioAvailUs = this._videoAvailUs = undefined; this._audio = !!this._params.audio; @@ -156,8 +158,8 @@ export class LatencyController { `Buffer is full, starting. Available ms=${this._availableUs / 1000}`, ); this._startThreshUs = 0; - if (this._startTime === 0) { - this._startTime = this._getCurTimeMs(); + if (this._startTimeMs === 0) { + this._startTimeMs = this._getCurTimeMs(); } } return res; @@ -180,7 +182,7 @@ export class LatencyController { this._availableUs = Math.min(this._availableUs, this._videoAvailUs); } - if (this._startTime > 0) { + if (this._startTimeMs > 0) { this._bufferMeter.update(this._availableUs / 1000, this._getCurTimeMs()); } } @@ -204,86 +206,71 @@ export class LatencyController { if (this._audioAvailUs < 0) this._audioAvailUs = 0; } - _seek(distUs) { - if (distUs <= 0) return; - - let tNow = this._getCurTimeMs(); - // if (speed > this._speed) { - if (this._lastSeekTime > 0 && tNow - this._lastSeekTime < 3000) return; - // } - this._lastSeekTime = tNow; - - this._logger.debug( - `Seek forward by ${distUs / 1000}ms, cur bufer ms=${this._availableUs / 1000}`, - ); - this._stateMgr.incCurrentTsSmp(this._audioConfig.tsUsToSmpCnt(distUs)); - if (this._video) this._videoAvailUs -= distUs; - if (this._audio) this._audioAvailUs -= distUs; - this._availableUs -= distUs; - } - _adjustPlaybackLatency() { - if (this._startTime === 0) return; // not started yet - - // this._logger.debug(`Available ms=${this._availableUs / 1000}, ${this._bufferMeter.get()}`); - // let availableMs = this._bufferMeter.get() / 1000; - // if (availableMs <= this._latencyMs * this._subHysteresis) { - // // this._setSpeed(1.0, availableMs); - // } else if (availableMs > this._latencyMs * this._hysteresis) { - // // this._setSpeed(1.1, availableMs); // speed boost - // this._seek((availableMs - this._latencyMs) * 1000); - // // this._bufferMeter.reset(); - // } + if (this._startTimeMs === 0) return; // not started yet const now = this._getCurTimeMs(); - const age = now - this._startTime; + const age = now - this._startTimeMs; const useShort = (age < this._warmupMs); const bufMin = useShort ? this._bufferMeter.short(now) : this._bufferMeter.long(now); const bufEma = this._bufferMeter.ema(); - const delta = bufMin - this._latencyMs; + const deltaMs = bufMin - this._latencyMs; // const stable = Math.abs(bufMin - bufEma) < (this._latencyMs * 0.1); // wait for holdMs to avoid acting on single spikes - let doAdjustment = false; - if (delta > 20) { + let goForward = false; + if (deltaMs > this._minLatencyDelta) { if (!this._pendingStableSince) { this._pendingStableSince = now; } else if (now - this._pendingStableSince > this._holdMs) { - doAdjustment = true; + goForward = true; } } else { this._pendingStableSince = null; } + this._adjustFn(goForward, deltaMs, bufMin, now); + } + + _startingBufferLevel() { + return 0.98 * this._latencyMs * 1000; + } + + _zap(goForward, deltaMs, bufMin, now) { let rate = 1.0; - if (doAdjustment) { + if (goForward) { if (now - this._lastActionTime < this._minRateChangeIntervalMs) { return; } - rate = clamp(1 + (this._rateK * delta), this._minRate, this._maxRate); + rate = clamp(1 + (this._rateK * deltaMs), this._minRate, this._maxRate); } + this._lastActionTime = now; - this._applyPlaybackRate(rate, bufMin, now); - } - - _startingBufferLevel() { - return 0.98 * this._latencyMs * 1000; + if (Math.abs(rate - 1.0) < 0.001) rate = 1.0; // snap + this._setSpeed(rate, bufMin); } - // _adjustPlaybackLatency(availableMs) { - // let targetLatencyMs = 1.1 * this._config.latency; - // if (availableMs > targetLatencyMs) { - // this._startTime -= availableMs - targetLatencyMs; - // } else if (availableMs < 0.2 * targetLatencyMs) { - // this._startTime += targetLatencyMs; - // } - // } + _seek(goForward, deltaMs, bufMin, now) { + if (!goForward) return; - _applyPlaybackRate(rate, availableMs, now) { - if (Math.abs(rate - 1.0) < 0.001) rate = 1.0; // snap - this._setSpeed(rate, availableMs); + if (goForward) { + if (now - this._lastActionTime < this._minSeekIntervalMs) { + return; + } + } this._lastActionTime = now; + + this._logger.debug( + `Seek forward by ${deltaMs}ms, cur bufer ms=${bufMin}`, + ); + + let deltaUs = deltaMs * 1000; + this._stateMgr.incCurrentTsSmp(this._audioConfig.tsUsToSmpCnt(deltaUs)); + if (this._video) this._videoAvailUs -= deltaUs; + if (this._audio) this._audioAvailUs -= deltaUs; + this._availableUs -= deltaUs; } + } diff --git a/src/nimio.js b/src/nimio.js index ad61f46..e4f4aca 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -501,6 +501,8 @@ export default class Nimio { sampleRate: sampleRate, stateSab: this._sab, latency: this._config.latency, + latencyTolerance: this._config.latencyTolerance, + latencyAdjustMethod: this._config.latencyAdjustMethod, idle: idle || false, videoEnabled: !this._noVideo, logLevel: this._config.logLevel, From 4679198be470da6edf0f0e5bec7c9494541d009c Mon Sep 17 00:00:00 2001 From: agonchar Date: Mon, 24 Nov 2025 16:19:23 +1000 Subject: [PATCH 12/67] Fix initial seek on startup. Adjust readable audio buffer step if there's not enough frames in a specified range. Fix no audio mode --- index.html | 1 + src/latency-controller.js | 69 +++++++++++++++------- src/latency/buffer-meter.js | 9 +-- src/media/buffers/readable-audio-buffer.js | 40 +++++++------ src/media/decoders/flow-audio.js | 1 + src/media/decoders/flow-video.js | 1 + src/nimio.js | 17 ++++-- src/shared/sliding-min.js | 3 +- src/shared/values.js | 3 + src/state-manager.js | 16 +++++ src/ui/debug-view.js | 1 + src/ui/ui.css | 4 +- 12 files changed, 112 insertions(+), 53 deletions(-) diff --git a/index.html b/index.html index 482df87..29bc69a 100644 --- a/index.html +++ b/index.html @@ -56,6 +56,7 @@ type: "input", }, workletLogs: true, + latencyAdjustMethod: "fast-forward", // "fast-forward" | "seek" }); window.nimio.on("nimio:play", (inst, contId) => { diff --git a/src/latency-controller.js b/src/latency-controller.js index 5113196..60a831c 100644 --- a/src/latency-controller.js +++ b/src/latency-controller.js @@ -8,19 +8,21 @@ export class LatencyController { this._instName = instName; this._stateMgr = stateMgr; this._audioConfig = audioConfig; + this._params = params; + this._audio = !!this._params.audio; + this._video = !!this._params.video; + this._latencyMs = this._params.latency; this._shortWindowMs = 300; this._longWindowMs = 1500; this._bufferMeter = new LatencyBufferMeter( instName, this._shortWindowMs, - this._longWindowMs + this._longWindowMs, ); this._adjustFn = params.adjustMethod === "seek" ? this._seek : this._zap; - this.reset(); - this._startThreshUs = this._startingBufferLevel(); this._minThreshUs = 50_000; // 50ms this._hysteresis = this._latencyMs < 1000 ? 1.5 : 1.25; @@ -28,7 +30,9 @@ export class LatencyController { this._warmupMs = 3000; this._holdMs = 500; + this._startHoldMs = 100; this._minRate = 0.95; + this._minRateStep = 1 / 128; this._maxRate = 1.1; this._rateK = 0.00015; // proportional gain: rate = 1 + rateK * deltaMs @@ -38,6 +42,8 @@ export class LatencyController { this._getCurTimeMs = currentTimeGetterMs(); + this.reset(); + this._logger = LoggersFactory.create(instName, "Latency ctrl", params.port); this._logger.debug( `initialized: latency=${this._latencyMs}ms, start threshold=${this._startThreshUs}us, video=${this._video}, audio=${this._audio}`, @@ -50,12 +56,10 @@ export class LatencyController { this._prevVideoTime = 0; this._bufferMeter.reset(); - this._startTimeMs = 0; + this._startTimeMs = -1; + this._lastActionTime = -this._minSeekIntervalMs; this._audioAvailUs = this._videoAvailUs = undefined; - - this._audio = !!this._params.audio; - this._video = !!this._params.video; - this._latencyMs = this._params.latency; + this._pendingStableSince = null; } start() { @@ -158,7 +162,7 @@ export class LatencyController { `Buffer is full, starting. Available ms=${this._availableUs / 1000}`, ); this._startThreshUs = 0; - if (this._startTimeMs === 0) { + if (this._startTimeMs < 0) { this._startTimeMs = this._getCurTimeMs(); } } @@ -182,7 +186,9 @@ export class LatencyController { this._availableUs = Math.min(this._availableUs, this._videoAvailUs); } - if (this._startTimeMs > 0) { + // this._logger.debug(`Available ms=${this._availableUs / 1000}, start time=${this._startTimeMs}`); + + if (this._startTimeMs >= 0) { this._bufferMeter.update(this._availableUs / 1000, this._getCurTimeMs()); } } @@ -207,21 +213,27 @@ export class LatencyController { } _adjustPlaybackLatency() { - if (this._startTimeMs === 0) return; // not started yet + if (this._startTimeMs < 0) return; // not started yet const now = this._getCurTimeMs(); const age = now - this._startTimeMs; - const useShort = (age < this._warmupMs); + const useShort = age < this._warmupMs; - const bufMin = useShort ? this._bufferMeter.short(now) : this._bufferMeter.long(now); - const bufEma = this._bufferMeter.ema(); + const shortB = this._bufferMeter.short(now); + this._stateMgr.setMinBufferMs("short", shortB); + const longB = this._bufferMeter.long(now); + this._stateMgr.setMinBufferMs("long", longB); + const emaB = this._bufferMeter.ema(); + this._stateMgr.setMinBufferMs("ema", emaB); + const bufMin = useShort ? shortB : longB; const deltaMs = bufMin - this._latencyMs; // const stable = Math.abs(bufMin - bufEma) < (this._latencyMs * 0.1); - // wait for holdMs to avoid acting on single spikes let goForward = false; if (deltaMs > this._minLatencyDelta) { + // this._logger.debug(`Delta ms=${deltaMs}, buffer ms=${bufMin}, age ms=${age}`); + // wait for holdMs to avoid acting on single spikes if (!this._pendingStableSince) { this._pendingStableSince = now; } else if (now - this._pendingStableSince > this._holdMs) { @@ -231,6 +243,11 @@ export class LatencyController { this._pendingStableSince = null; } + if (this._lastActionTime < 0) { + // Do seek on startup if possible + if (this._tryInitialSeek(deltaMs, bufMin, now)) return; + } + this._adjustFn(goForward, deltaMs, bufMin, now); } @@ -238,17 +255,28 @@ export class LatencyController { return 0.98 * this._latencyMs * 1000; } + _tryInitialSeek(deltaMs, buf, now) { + if (deltaMs > this._latencyMs) { + let stableTime = now - this._pendingStableSince; + if (stableTime >= this._startHoldMs && stableTime < this._warmupMs) { + this._seek(true, deltaMs, buf, now); + return true; + } + } + return false; + } + _zap(goForward, deltaMs, bufMin, now) { let rate = 1.0; if (goForward) { if (now - this._lastActionTime < this._minRateChangeIntervalMs) { return; } - rate = clamp(1 + (this._rateK * deltaMs), this._minRate, this._maxRate); + rate = clamp(1 + this._rateK * deltaMs, this._minRate, this._maxRate); + this._lastActionTime = now; } - this._lastActionTime = now; - if (Math.abs(rate - 1.0) < 0.001) rate = 1.0; // snap + if (Math.abs(rate - 1.0) < this._minRateStep) rate = 1.0; // snap this._setSpeed(rate, bufMin); } @@ -262,9 +290,7 @@ export class LatencyController { } this._lastActionTime = now; - this._logger.debug( - `Seek forward by ${deltaMs}ms, cur bufer ms=${bufMin}`, - ); + this._logger.debug(`Seek forward by ${deltaMs}ms, cur bufer ms=${bufMin}`); let deltaUs = deltaMs * 1000; this._stateMgr.incCurrentTsSmp(this._audioConfig.tsUsToSmpCnt(deltaUs)); @@ -272,5 +298,4 @@ export class LatencyController { if (this._audio) this._audioAvailUs -= deltaUs; this._availableUs -= deltaUs; } - } diff --git a/src/latency/buffer-meter.js b/src/latency/buffer-meter.js index 2c3b892..061c1a0 100644 --- a/src/latency/buffer-meter.js +++ b/src/latency/buffer-meter.js @@ -36,7 +36,7 @@ export class LatencyBufferMeter { estimatedBuffer(timeMs) { const shortB = this._shortMin.getMin(timeMs); - const longB = this._longMin.getMin(timeMs); + const longB = this._longMin.getMin(timeMs); return 0.6 * shortB + 0.4 * longB; } @@ -52,7 +52,8 @@ export class LatencyBufferMeter { else this._ema = this._emaAlpha * value + (1 - this._emaAlpha) * this._ema; } - _fpw(sizeMs) { // max frames per window - return Math.ceil(60 * sizeMs / 1000); + _fpw(sizeMs) { + // max frames per window + return Math.ceil((60 * sizeMs) / 1000); } -} \ No newline at end of file +} diff --git a/src/media/buffers/readable-audio-buffer.js b/src/media/buffers/readable-audio-buffer.js index f280f6c..30652ee 100644 --- a/src/media/buffers/readable-audio-buffer.js +++ b/src/media/buffers/readable-audio-buffer.js @@ -66,6 +66,22 @@ export class ReadableAudioBuffer extends SharedAudioBuffer { } _fillOutput(outputChannels, startIdx, endIdx, startOffset, endOffset, step) { + let outLength = outputChannels[0].length; + if (startIdx !== null && endIdx !== null) { + let expProcCnt = endOffset - startOffset; + if (startIdx !== endIdx) { + expProcCnt = this.sampleCount - startOffset + endOffset; + } + let steppedCount = (expProcCnt / step + 0.5) >>> 0; + if (steppedCount < outLength) { + console.log( + `Fixed step to ${expProcCnt / outLength} from ${step}, expected count: ${expProcCnt}, stepped count: ${steppedCount}, accurate stepped cnt: ${expProcCnt / step}`, + ); + step = expProcCnt / outLength; + if (step < 0.95) step = 0.95; + } + } + let processed = null; if (startIdx === endIdx) { if (startIdx === null) { @@ -115,30 +131,20 @@ export class ReadableAudioBuffer extends SharedAudioBuffer { } if (startCount === null) { - console.error( - "Fill silence at the start", - outputChannels[0].length - endCount, - ); - this._fillSilence(outputChannels, 0, outputChannels[0].length - endCount); + console.error("Fill silence at the start", outLength - endCount); + this._fillSilence(outputChannels, 0, outLength - endCount); } else if (endCount === null) { - console.error( - "Fill silence at the end", - outputChannels[0].length - startCount, - ); - this._fillSilence( - outputChannels, - startCount, - outputChannels[0].length - startCount, - ); - } else if (startCount + endCount < outputChannels[0].length) { + console.error("Fill silence at the end", outLength - startCount); + this._fillSilence(outputChannels, startCount, outLength - startCount); + } else if (startCount + endCount < outLength) { console.error( "Fill silence in the middle", - outputChannels[0].length - startCount - endCount, + outLength - startCount - endCount, ); this._fillSilence( outputChannels, startCount, - outputChannels[0].length - startCount - endCount, + outLength - startCount - endCount, ); } diff --git a/src/media/decoders/flow-audio.js b/src/media/decoders/flow-audio.js index 3cdca9e..7fdca6c 100644 --- a/src/media/decoders/flow-audio.js +++ b/src/media/decoders/flow-audio.js @@ -14,6 +14,7 @@ export class DecoderFlowAudio extends DecoderFlow { async _handleDecoderOutput(frame, data) { if (await this._handleDecodedFrame(frame)) { + if (!this._buffer) return; this._state.setAudioLatestTsUs(this._buffer.lastFrameTs); } this._state.setAudioDecoderQueue(data.decoderQueue); diff --git a/src/media/decoders/flow-video.js b/src/media/decoders/flow-video.js index 9f40b5e..316dc08 100644 --- a/src/media/decoders/flow-video.js +++ b/src/media/decoders/flow-video.js @@ -12,6 +12,7 @@ export class DecoderFlowVideo extends DecoderFlow { async _handleDecoderOutput(frame, data) { if (await this._handleDecodedFrame(frame)) { + if (!this._buffer) return; this._state.setVideoLatestTsUs(this._buffer.lastFrameTs); } this._state.setVideoDecoderQueue(data.decoderQueue); diff --git a/src/nimio.js b/src/nimio.js index e4f4aca..3dd9343 100644 --- a/src/nimio.js +++ b/src/nimio.js @@ -353,12 +353,15 @@ export default class Nimio { } } - if (this._noAudio || this._videoBuffer.getTimeCapacity() >= 0.5) { - if (!this._audioContext) { - this._setPlaybackStartTs("video"); - } + if ( + this._noAudio || + (!this._audioContext && this._videoBuffer.getTimeCapacity() >= 0.5) + ) { + this._setPlaybackStartTs("video"); if (!this._noAudio && this._audioCtxProvider.isRunning()) { + // it doesn't make sense to start no audio mode via audio worklet + // if audio context is suspended await this._startNoAudioMode(); } } @@ -472,6 +475,7 @@ export default class Nimio { async _initAudioProcessor(sampleRate, channels, idle) { if (!this._audioContext) { this._audioCtxProvider.init(sampleRate); + this._logger.debug("Init audio processor", sampleRate, channels); this._audioCtxProvider.setChannelCount(channels); this._audioContext = this._audioCtxProvider.get(); @@ -542,14 +546,15 @@ export default class Nimio { _setNoVideo(yes) { if (yes === undefined) yes = true; - this._noVideo = !!yes; + this._noVideo = yes; this._latencyCtrl.videoEnabled = !yes; } _setNoAudio(yes) { if (yes === undefined) yes = true; - this._noAudio = !!yes; + this._noAudio = yes; this._latencyCtrl.audioEnabled = !yes; + this._logger.debug("Set no audio:", yes); } async _startNoAudioMode() { diff --git a/src/shared/sliding-min.js b/src/shared/sliding-min.js index d80203a..8287d42 100644 --- a/src/shared/sliding-min.js +++ b/src/shared/sliding-min.js @@ -21,7 +21,7 @@ export class SlidingMin { } getMin(timeMs) { - this._expire(timeMs) + this._expire(timeMs); if (this._vals.length === 0) return null; return this._vals.front; @@ -41,4 +41,3 @@ export class SlidingMin { } } } - \ No newline at end of file diff --git a/src/shared/values.js b/src/shared/values.js index ff01571..231fc05 100644 --- a/src/shared/values.js +++ b/src/shared/values.js @@ -15,4 +15,7 @@ export const IDX = { PLAYBACK_START_TS: [9, 10], VIDEO_LATEST_TS: [11, 12], AUDIO_LATEST_TS: [13, 14], + MIN_BUFFER_SHORT: 15, + MIN_BUFFER_LONG: 16, + MIN_BUFFER_EMA: 17, }; diff --git a/src/state-manager.js b/src/state-manager.js index 5d6ad0b..9feb049 100644 --- a/src/state-manager.js +++ b/src/state-manager.js @@ -131,6 +131,22 @@ export class StateManager { Atomics.store(this._flags, IDX.AUDIO_DECODER_QUEUE, f); } + getMinBufferMs(type) { + return Atomics.load(this._flags, this._bufTypeIdx(type)); + } + + setMinBufferMs(type, val) { + return Atomics.store(this._flags, this._bufTypeIdx(type), val); + } + + _bufTypeIdx(type) { + return type === "short" + ? IDX.MIN_BUFFER_SHORT + : type === "long" + ? IDX.MIN_BUFFER_LONG + : IDX.MIN_BUFFER_EMA; + } + _atomicLoad64(idxs) { const idx = idxs[0]; while (true) { diff --git a/src/ui/debug-view.js b/src/ui/debug-view.js index 09fa1de..dfb612f 100644 --- a/src/ui/debug-view.js +++ b/src/ui/debug-view.js @@ -34,6 +34,7 @@ export class DebugView { let aDecQueue = this._state.getAudioDecoderQueue(); this._inst.textContent = + `Buffer(l/s/e):.${this._state.getMinBufferMs("long").toString().padStart(4, ".")}/${this._state.getMinBufferMs("short").toString()}/${this._state.getMinBufferMs("ema").toString()}ms \n` + `Video buffer:....${this._vBuffer.length.toString().padStart(4, ".")}f..${videoMs}ms \n` + `Audio buffer:..........${audioMs.toString().padStart(4, ".")}ms \n` + `Silence inserted:......${Math.ceil(silenceMs).toString().padStart(4, ".")}ms \n` + diff --git a/src/ui/ui.css b/src/ui/ui.css index 182be24..55a1b7f 100644 --- a/src/ui/ui.css +++ b/src/ui/ui.css @@ -71,8 +71,8 @@ position: absolute; top: 0; left: 0; - width: 220px; - height: 112px; + width: 230px; + height: 140px; opacity: 0.7; background-color: white; white-space: pre-line; From 237a3e369ddd95c7f531577baa191cac93a21cba Mon Sep 17 00:00:00 2001 From: agonchar Date: Mon, 24 Nov 2025 16:51:33 +1000 Subject: [PATCH 13/67] Fix test --- src/media/buffers/readable-audio-buffer.js | 6 +++--- tests/readable-audio-buffer.test.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/media/buffers/readable-audio-buffer.js b/src/media/buffers/readable-audio-buffer.js index 30652ee..c309292 100644 --- a/src/media/buffers/readable-audio-buffer.js +++ b/src/media/buffers/readable-audio-buffer.js @@ -74,11 +74,11 @@ export class ReadableAudioBuffer extends SharedAudioBuffer { } let steppedCount = (expProcCnt / step + 0.5) >>> 0; if (steppedCount < outLength) { - console.log( - `Fixed step to ${expProcCnt / outLength} from ${step}, expected count: ${expProcCnt}, stepped count: ${steppedCount}, accurate stepped cnt: ${expProcCnt / step}`, - ); step = expProcCnt / outLength; if (step < 0.95) step = 0.95; + console.log( + `Fixed step to ${step}, expected count: ${expProcCnt}, stepped count: ${steppedCount}`, + ); } } diff --git a/tests/readable-audio-buffer.test.js b/tests/readable-audio-buffer.test.js index b479c81..6dc3247 100644 --- a/tests/readable-audio-buffer.test.js +++ b/tests/readable-audio-buffer.test.js @@ -207,7 +207,7 @@ describe("ReadableAudioBuffer", () => { rab.frames[1].fill(0.6); rab.setWriteIdx(2); - const outLength = rab.sampleCount; + const outLength = rab.sampleCount * 4; const out = [new Float32Array(outLength), new Float32Array(outLength)]; const step = 4; const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); From 06376882241e43ad3b5ebb842eedcec9c4f06dda Mon Sep 17 00:00:00 2001 From: agonchar Date: Wed, 26 Nov 2025 18:33:10 +1000 Subject: [PATCH 14/67] Fix inspection faults --- index.html | 3 ++- src/latency-controller.js | 14 +++++++++++--- src/media/buffers/writable-audio-buffer.js | 2 +- src/media/decoders/timestamp-manager.js | 11 +++-------- src/player-config.js | 4 ++-- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/index.html b/index.html index 29bc69a..813a707 100644 --- a/index.html +++ b/index.html @@ -12,7 +12,7 @@ style="width: 408px; height: 30px; font-size: 16px" class="nimio-input" placeholder="ws:// or wss:// URL of the stream" - value="ws://172.16.80.128:8081/live/stream" + value="wss://demo-nimble.softvelum.com/live/bbb" />