diff --git a/Source/WebCore/platform/graphics/SourceBufferPrivate.cpp b/Source/WebCore/platform/graphics/SourceBufferPrivate.cpp index 49a6a90fd518e..07ddc085533f6 100644 --- a/Source/WebCore/platform/graphics/SourceBufferPrivate.cpp +++ b/Source/WebCore/platform/graphics/SourceBufferPrivate.cpp @@ -932,6 +932,14 @@ void SourceBufferPrivate::didReceiveSample(Ref&& originalSample) erasedSamples.addRange(iterPair.first, iterPair.second); } + // There are many files out there where the frame times are not perfectly contiguous and may have small overlaps + // between the beginning of a frame and the end of the previous one; therefore a tolerance is needed whenever + // durations are considered. + // For instance, most WebM files are muxed rounded to the millisecond (the default TimecodeScale of the format) + // but their durations use a finer timescale (causing a sub-millisecond overlap). More rarely, there are also + // MP4 files with slightly off tfdt boxes, presenting a similar problem at the beginning of each fragment. + const MediaTime contiguousFrameTolerance = MediaTime(1, 1000); + // When appending media containing B-frames (media whose samples' presentation timestamps // do not increase monotonically, the prior erase steps could leave samples in the trackBuffer // which will be disconnected from its previous I-frame. If the incoming frame is an I-frame, @@ -954,19 +962,36 @@ void SourceBufferPrivate::didReceiveSample(Ref&& originalSample) while (nextSyncSample != trackBuffer.samples().decodeOrder().end() && nextSyncSample->second->presentationTime() <= sample->presentationTime()) nextSyncSample = trackBuffer.samples().decodeOrder().findSyncSampleAfterDecodeIterator(nextSyncSample); - INFO_LOG(LOGIDENTIFIER, "Discovered out-of-order frames, from: ", *nextSampleInDecodeOrder->second.get(), " to: ", (nextSyncSample == trackBuffer.samples().decodeOrder().end() ? "[end]"_s : toString(*nextSyncSample->second.get()))); + if (nextSampleInDecodeOrder->second->presentationTime() < sample->presentationTime()) { + // Try to fix the out-of-ordering by placing the decoding timestamp of sample after the decoding timestamp + // of the last pre-existing sample before the next sync sample whis has a presentationTime lower than sample. + auto lastSampleToErase = trackBuffer.samples().decodeOrder().rbegin(); + if (nextSyncSample != trackBuffer.samples().decodeOrder().end()) { + lastSampleToErase = trackBuffer.samples().decodeOrder().reverseFindSampleWithDecodeKey(nextSyncSample->first); + lastSampleToErase++; + } + + if (lastSampleToErase != trackBuffer.samples().decodeOrder().rend()) { + if (lastSampleToErase->second->presentationTime() < sample->presentationTime()) { + const MediaTime epsilon = MediaTime(100, 1000000); // 100 µs. + auto safeDecodeTime = lastSampleToErase->second->decodeTime() + epsilon; + if (safeDecodeTime > sample->decodeTime() + && safeDecodeTime < (sample->decodeTime() + sample->duration() - contiguousFrameTolerance - contiguousFrameTolerance)) { + INFO_LOG(LOGIDENTIFIER, "Discovered out-of-order frames, from: ", *nextSampleInDecodeOrder->second, " to: ", (nextSyncSample == trackBuffer.samples().decodeOrder().end() ? "[end]"_s : toString(*nextSyncSample->second.get())), + ", but fixed the ordering by changing sample DTS from ", sample->decodeTime(), " to ", safeDecodeTime); + sample->setTimestamps(sample->presentationTime(), safeDecodeTime); + break; + } + } + } + } + + INFO_LOG(LOGIDENTIFIER, "Discovered out-of-order frames, from: ", *nextSampleInDecodeOrder->second, + " to: ", (nextSyncSample == trackBuffer.samples().decodeOrder().end() ? "[end]"_s : toString(*nextSyncSample->second.get())), + "erasing them"); erasedSamples.addRange(nextSampleInDecodeOrder, nextSyncSample); } while (false); - // There are many files out there where the frame times are not perfectly contiguous and may have small overlaps - // between the beginning of a frame and the end of the previous one; therefore a tolerance is needed whenever - // durations are considered. - // For instance, most WebM files are muxed rounded to the millisecond (the default TimecodeScale of the format) - // but their durations use a finer timescale (causing a sub-millisecond overlap). More rarely, there are also - // MP4 files with slightly off tfdt boxes, presenting a similar problem at the beginning of each fragment. - // Same as tolerance in SourceBuffer::canPlayThroughRange(). - const MediaTime contiguousFrameTolerance = MediaTime(1, 1000); - // If highest presentation timestamp for track buffer is set and less than or equal to presentation timestamp if (trackBuffer.highestPresentationTimestamp().isValid() && trackBuffer.highestPresentationTimestamp() - contiguousFrameTolerance <= presentationTimestamp) { // Remove all coded frames from track buffer that have a presentation timestamp greater than highest diff --git a/Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.cpp b/Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.cpp index 226af18e9e42e..6c28f287be51c 100644 --- a/Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.cpp @@ -468,20 +468,15 @@ void AppendPipeline::appsinkNewSample(const Track& track, GRefPtr&& s GstSegment* segment = gst_sample_get_segment(sample.get()); auto mediaSample = MediaSampleGStreamer::create(WTFMove(sample), track.presentationSize, track.trackId); - /* this logic does not work when we insert piece of stream with different segment start - and then return back to original offset. It causes the inserted segment is shifted/overlapped. - if (segment && (segment->time || segment->start)) { // MP4 has the concept of edit lists, where some buffer time needs to be offsetted, often very slightly, // to get exact timestamps. - MediaTime pts = bufferTimeToStreamTime(segment, GST_BUFFER_PTS(buffer)); MediaTime dts = bufferTimeToStreamTime(segment, GST_BUFFER_DTS(buffer)); GST_TRACE_OBJECT(track.appsinkPad.get(), "Mapped buffer to segment, PTS %" GST_TIME_FORMAT " -> %s DTS %" GST_TIME_FORMAT " -> %s", GST_TIME_ARGS(GST_BUFFER_PTS(buffer)), pts.toString().utf8().data(), GST_TIME_ARGS(GST_BUFFER_DTS(buffer)), dts.toString().utf8().data()); mediaSample->setTimestamps(pts, dts); - - } else */ if (!GST_BUFFER_DTS(buffer) && GST_BUFFER_PTS(buffer) > 0 && GST_BUFFER_PTS(buffer) <= 100'000'000) { + } else if (!GST_BUFFER_DTS(buffer) && GST_BUFFER_PTS(buffer) > 0 && GST_BUFFER_PTS(buffer) <= 100'000'000) { // Because a track presentation time starting at some close to zero, but not exactly zero time can cause unexpected // results for applications, we extend the duration of this first sample to the left so that it starts at zero. // This is relevant for files that should have an edit list but don't, or when using GStreamer < 1.16, where