From 44be8d5bcd1a8b0f30335d50b51f7cb9e6d9a0c0 Mon Sep 17 00:00:00 2001 From: dimitris-c Date: Sat, 11 Oct 2025 23:41:23 +0300 Subject: [PATCH 1/4] Adds mp4 restructure improvements --- .../Audio Source/FileAudioSource.swift | 87 +++++++++++++++---- .../Audio Source/Mp4/Mp4Restructure.swift | 70 ++++++++++----- .../Mp4/RemoteMp4Restructure.swift | 24 ++--- 3 files changed, 128 insertions(+), 53 deletions(-) diff --git a/AudioStreaming/Streaming/Audio Source/FileAudioSource.swift b/AudioStreaming/Streaming/Audio Source/FileAudioSource.swift index 070b9bf..d15551c 100644 --- a/AudioStreaming/Streaming/Audio Source/FileAudioSource.swift +++ b/AudioStreaming/Streaming/Audio Source/FileAudioSource.swift @@ -120,11 +120,15 @@ final class FileAudioSource: NSObject, CoreAudioStreamSource { if isMp4, !mp4IsAlreadyOptimized { if !mp4Restructure.dataOptimized { do { - if let mp4OptimizeInfo = try mp4Restructure.checkIsOptimized(data: data) { - try performMp4Restructure(inputStream: inputStream, mp4OptimizeInfo: mp4OptimizeInfo) - } else { + switch try mp4Restructure.checkIsOptimized(data: data) { + case .undetermined: + // Not enough bytes yet; wait for more data before deciding + break + case .optimized: mp4IsAlreadyOptimized = true delegate?.dataAvailable(source: self, data: data) + case let .needsRestructure(moovOffset): + try performMp4Restructure(inputStream: inputStream, moovOffset: moovOffset) } } catch { delegate?.errorOccurred(source: self, error: error) @@ -141,24 +145,71 @@ final class FileAudioSource: NSObject, CoreAudioStreamSource { } } - func performMp4Restructure(inputStream: InputStream, mp4OptimizeInfo: Mp4OptimizeInfo) throws { - let offsetAccepted = inputStream.setProperty(mp4OptimizeInfo.moovOffset, forKey: .fileCurrentOffsetKey) - if offsetAccepted { - let moovDataBuffer = UnsafeMutablePointer.uint8pointer(of: mp4OptimizeInfo.moovSize) - defer { moovDataBuffer.deallocate() } - let moovRead = inputStream.read(moovDataBuffer, maxLength: mp4OptimizeInfo.moovSize) - if moovRead > 0 { - let data = Data(bytes: moovDataBuffer, count: moovRead) - let moovData = try mp4Restructure.restructureMoov(data: data) - delegate?.dataAvailable(source: self, data: moovData.initialData) - if !inputStream.setProperty(moovData.mdatOffset, forKey: .fileCurrentOffsetKey) { + func performMp4Restructure(inputStream: InputStream, moovOffset: Int) throws { + let offsetAccepted = inputStream.setProperty(moovOffset, forKey: .fileCurrentOffsetKey) + if !offsetAccepted { + delegate?.errorOccurred(source: self, error: inputStream.streamError ?? AudioSystemError.playerStartError) + return + } + + // Read moov header (8 bytes) + var header = [UInt8](repeating: 0, count: 8) + let headerRead = inputStream.read(&header, maxLength: 8) + guard headerRead == 8 else { + delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError) + return + } + + // Parse size and type (big endian) + let size32 = Data(header[0 ..< 4]).withUnsafeBytes { $0.load(as: UInt32.self) }.bigEndian + let type32 = Data(header[4 ..< 8]).withUnsafeBytes { $0.load(as: UInt32.self) }.bigEndian + guard Int(type32) == Atoms.moov else { + delegate?.errorOccurred(source: self, error: Mp4RestructureError.missingMoovAtom) + return + } + + var moovSize = Int(size32) + var moovData = Data(header) + + // Extended size (64-bit) + if moovSize == 1 { + var ext = [UInt8](repeating: 0, count: 8) + let extRead = inputStream.read(&ext, maxLength: 8) + guard extRead == 8 else { + delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError) + return + } + let ext64 = Data(ext).withUnsafeBytes { $0.load(as: UInt64.self) }.bigEndian + moovSize = Int(ext64) + moovData.append(contentsOf: ext) + } + + let remaining = moovSize - moovData.count + if remaining < 0 { + delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError) + return + } + if remaining > 0 { + var buffer = [UInt8](repeating: 0, count: remaining) + var total = 0 + while total < remaining { + let readBytes = buffer.withUnsafeMutableBytes { ptr -> Int in + let base = ptr.baseAddress!.assumingMemoryBound(to: UInt8.self).advanced(by: total) + return inputStream.read(base, maxLength: remaining - total) + } + guard readBytes > 0 else { delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError) + return } - } else { - delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError) + total += readBytes } - } else { - delegate?.errorOccurred(source: self, error: inputStream.streamError ?? AudioSystemError.playerStartError) + moovData.append(contentsOf: buffer) + } + + let moovResult = try mp4Restructure.restructureMoov(data: moovData) + delegate?.dataAvailable(source: self, data: moovResult.initialData) + if !inputStream.setProperty(moovResult.mdatOffset, forKey: .fileCurrentOffsetKey) { + delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError) } } diff --git a/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift b/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift index 04616fa..9eed1e1 100644 --- a/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift +++ b/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift @@ -36,7 +36,7 @@ enum Atoms { static var cmov: Int { fourCcToInt("cmov") } static var stco: Int { fourCcToInt("stco") } - static var co64: Int { fourCcToInt("c064") } + static var co64: Int { fourCcToInt("co64") } static var atomPreampleSize: Int = 8 @@ -75,6 +75,12 @@ enum Mp4RestructureError: Error { case networkError(Error) } +enum OptimizeCheckResult: Equatable { + case optimized + case needsRestructure(moovOffset: Int) + case undetermined +} + final class Mp4Restructure { private var atomOffset: Int = 0 @@ -129,24 +135,36 @@ final class Mp4Restructure { return (initialData, mdatOffset) } - /// Returns `nil` if the data is optimized otherwise `Mp4OptimizeInfo` - func checkIsOptimized(data: Data) throws -> Mp4OptimizeInfo? { - while atomOffset < UInt64(data.count) { - var atomSize = try Int(getInteger(data: data, offset: atomOffset) as UInt32) - let atomType = try Int(getInteger(data: data, offset: atomOffset + 4) as UInt32) + /// Incrementally checks if the MP4 is optimized. Returns tri-state result. + func checkIsOptimized(data: Data) throws -> OptimizeCheckResult { + while atomOffset + 8 <= data.count { + var atomSize: Int = try Int(getInteger(data: data, offset: atomOffset) as UInt32) + let atomType: Int = try Int(getInteger(data: data, offset: atomOffset + 4) as UInt32) + var headerSize = 8 + + // Handle extended size (64-bit) + if atomSize == 1 { + if atomOffset + 16 > data.count { break } + let ext: UInt64 = try getInteger(data: data, offset: atomOffset + 8) + atomSize = Int(ext) + headerSize = 16 + } else if atomSize == 0 { + // Size extends to EOF; with partial data we can't determine full box + break + } + + // Bounds and sanity checks + if atomSize < headerSize || atomOffset + atomSize > data.count { break } + switch atomType { case Atoms.ftyp: - let ftypData = data[Int(atomOffset) ..< atomSize] + let start = atomOffset + let end = atomOffset + atomSize + let ftypData = data[start ..< end] let ftyp = MP4Atom(type: atomType, size: atomSize, offset: atomOffset, data: ftypData) self.ftyp = ftyp atoms.append(ftyp) case Atoms.mdat: - // ref: https://developer.apple.com/documentation/quicktime-file-format/movie_data_atom - // This atom can be quite large, and may exceed 2^32 bytes, in which case the size field will be set to 1, - // and the header will contain a 64-bit extended size field. - if atomSize == 1 { - atomSize = Int(try getInteger(data: data, offset: atomOffset + 8) as UInt64) - } let mdat = MP4Atom(type: atomType, size: atomSize, offset: atomOffset) atoms.append(mdat) foundMdat = true @@ -158,19 +176,21 @@ final class Mp4Restructure { let atom = MP4Atom(type: atomType, size: atomSize, offset: atomOffset) atoms.append(atom) } + if ftyp != nil { if foundMoov && !foundMdat { Logger.debug("🕵️ detected an optimized mp4", category: .generic) - return nil + return .optimized } else if !foundMoov && foundMdat { - Logger.debug("🕵️ detected an non-optimized mp4", category: .generic) - let possibleMoovOffset = Int(atomOffset) + atomSize - return Mp4OptimizeInfo(moovOffset: possibleMoovOffset, moovSize: atomSize) + Logger.debug("🕵️ detected a non-optimized mp4", category: .generic) + let possibleMoovOffset = atomOffset + atomSize + return .needsRestructure(moovOffset: possibleMoovOffset) } } + atomOffset += atomSize } - return nil + return .undetermined } /// logic taken from qt-faststart.c over at ffmpeg @@ -236,6 +256,8 @@ final class Mp4Restructure { // the next integer determines the `Number of entries` // https://developer.apple.com/documentation/quicktime-file-format/chunk_offset_atom/number_of_entries let numberOfOffsetEntries = try Int(moovAtom.getInteger() as UInt32) + // Adjust by moov size + let adjustDelta = moovAtomSize if atomType == Atoms.stco { Logger.debug("🏗️ patching stco atom...", category: .generic) if moovAtom.bytesAvailable < numberOfOffsetEntries * 4 { @@ -246,7 +268,7 @@ final class Mp4Restructure { for _ in 0 ..< numberOfOffsetEntries { let currentOffset = try Int(moovAtom.getInteger(moovAtom.offset) as UInt32) // adjust the offset by adding the size of moov atom - let adjustOffset = currentOffset + moovAtomSize + let adjustOffset = currentOffset + adjustDelta if currentOffset < 0, adjustOffset >= 0 { throw Mp4RestructureError.unableToRestructureData @@ -261,8 +283,8 @@ final class Mp4Restructure { } for _ in 0 ..< numberOfOffsetEntries { let currentOffset: Int = try moovAtom.getInteger(moovAtom.offset) - // adjust the offset by adding the size of moov atom - moovAtom.put(currentOffset + moovAtomSize) + // adjust the offset by adding the size of moov atom (write as big-endian 64-bit) + moovAtom.put(UInt64(currentOffset + adjustDelta).bigEndian) } } } @@ -271,10 +293,10 @@ final class Mp4Restructure { func getInteger(data: Data, offset: Int) throws -> T { let sizeOfInteger = MemoryLayout.size - guard sizeOfInteger <= data.count else { + guard offset >= 0, offset + sizeOfInteger <= data.count else { throw ByteBuffer.Error.eof } - let _offset = offset + sizeOfInteger - return T(data: data[_offset - sizeOfInteger ..< _offset]).bigEndian + let end = offset + sizeOfInteger + return T(data: data[offset ..< end]).bigEndian } } diff --git a/AudioStreaming/Streaming/Audio Source/Mp4/RemoteMp4Restructure.swift b/AudioStreaming/Streaming/Audio Source/Mp4/RemoteMp4Restructure.swift index 474d1be..b51ed0e 100644 --- a/AudioStreaming/Streaming/Audio Source/Mp4/RemoteMp4Restructure.swift +++ b/AudioStreaming/Streaming/Audio Source/Mp4/RemoteMp4Restructure.swift @@ -75,8 +75,15 @@ final class RemoteMp4Restructure { } self.audioData.append(data) do { - let value = try self.mp4Restructure.checkIsOptimized(data: self.audioData) - if let value { + switch try self.mp4Restructure.checkIsOptimized(data: self.audioData) { + case .undetermined: + break // keep streaming until decision can be made + case .optimized: + self.audioData = Data() + self.task?.cancel() + self.task = nil + completion(.success(nil)) + case let .needsRestructure(moovOffset): guard response.response?.statusCode == 206 else { Logger.error("⛔️ mp4 error: no moov before mdat and the stream is not seekable", category: .networking) completion(.failure(Mp4RestructureError.nonOptimizedMp4AndServerCannotSeek)) @@ -86,22 +93,15 @@ final class RemoteMp4Restructure { self.audioData = Data() self.task?.cancel() self.task = nil - self.fetchAndRestructureMoovAtom(offset: value.moovOffset) { result in + self.fetchAndRestructureMoovAtom(offset: moovOffset) { result in switch result { case let .success(value): - let data = value.data - let offset = value.offset self.dataOptimized = true - completion(.success(RestructuredData(initialData: data, mdatOffset: offset))) + completion(.success(RestructuredData(initialData: value.data, mdatOffset: value.offset))) case let .failure(error): completion(.failure(Mp4RestructureError.networkError(error))) } } - } else { - self.audioData = Data() - self.task?.cancel() - self.task = nil - completion(.success(nil)) } } catch { completion(.failure(Mp4RestructureError.invalidAtomSize)) @@ -132,6 +132,8 @@ final class RemoteMp4Restructure { } } + // removed warmup range helper + private func urlForPartialContent(with url: URL, offset: Int) -> URLRequest { var urlRequest = URLRequest(url: url) urlRequest.networkServiceType = .avStreaming From f0e1bff98a5433c97837a325e3fcecce77961672 Mon Sep 17 00:00:00 2001 From: dimitris-c Date: Mon, 13 Oct 2025 14:52:29 +0300 Subject: [PATCH 2/4] fixes data race --- .../Streaming/AudioPlayer/AudioPlayer.swift | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index 6204639..0cdb9de 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -651,18 +651,22 @@ open class AudioPlayer { guard playerContext.internalState != .paused else { return } + let snapshot = playerContext.entriesLock.withLock { + (reading: playerContext.audioReadingEntry, playing: playerContext.audioPlayingEntry) + } + if playerContext.internalState == .pendingNext { let entry = entriesQueue.dequeue(type: .upcoming) playerContext.setInternalState(to: .waitingForData) setCurrentReading(entry: entry, startPlaying: true, shouldClearQueue: true) rendererContext.resetBuffers() - } else if let playingEntry = playerContext.audioPlayingEntry, + } else if let playingEntry = snapshot.playing, playingEntry.seekRequest.requested, - playingEntry != playerContext.audioReadingEntry + playingEntry != snapshot.reading { playingEntry.audioStreamState.processedDataFormat = false playingEntry.reset() - if let readingEntry = playerContext.audioReadingEntry { + if let readingEntry = snapshot.reading { readingEntry.delegate = nil readingEntry.close() } @@ -677,20 +681,20 @@ open class AudioPlayer { setCurrentReading(entry: playingEntry, startPlaying: true, shouldClearQueue: false) } - } else if playerContext.audioReadingEntry == nil { + } else if snapshot.reading == nil { if entriesQueue.count(for: .upcoming) > 0 { let entry = entriesQueue.dequeue(type: .upcoming) - let shouldStartPlaying = playerContext.audioPlayingEntry == nil + let shouldStartPlaying = snapshot.playing == nil playerContext.setInternalState(to: .waitingForData) setCurrentReading(entry: entry, startPlaying: shouldStartPlaying, shouldClearQueue: false) - } else if playerContext.audioPlayingEntry == nil { + } else if snapshot.playing == nil { if playerContext.internalState != .stopped { stopEngine(reason: .eof) } } } - if let playingEntry = playerContext.audioPlayingEntry, + if let playingEntry = snapshot.playing, playingEntry.audioStreamState.processedDataFormat, playingEntry.calculatedBitrate() > 0.0 { From f1056c39756c07613c1da260ca5d45ea21f51770 Mon Sep 17 00:00:00 2001 From: dimitris-c Date: Mon, 13 Oct 2025 17:26:32 +0300 Subject: [PATCH 3/4] fix incorrect parsing of formatList --- .../Processors/AudioFileStreamProcessor.swift | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index f12f9e8..d8b46fd 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -337,27 +337,34 @@ final class AudioFileStreamProcessor { } private func processFormatList(entry: AudioEntry, fileStream: AudioFileStreamID) { - entry.lock.lock(); defer { entry.lock.unlock() } let info = fileStreamGetPropertyInfo(fileStream: fileStream, propertyId: kAudioFileStreamProperty_FormatList) - guard info.status == noErr else { return } - var list: [AudioFormatListItem] = Array(repeating: AudioFormatListItem(), count: Int(info.size)) - var size = UInt32(info.size) + guard info.status == noErr, info.size > 0 else { return } + + let itemStride = MemoryLayout.stride + let itemCount = Int(info.size) / itemStride + guard itemCount > 0 else { return } + + var list = [AudioFormatListItem](repeating: AudioFormatListItem(), count: itemCount) + var size = UInt32(itemCount * itemStride) AudioFileStreamGetProperty(fileStream, kAudioFileStreamProperty_FormatList, &size, &list) - let step = MemoryLayout.size - var i = 0 - while i * step < size { + + var chosenASBD: AudioStreamBasicDescription? + for i in 0.. Date: Mon, 13 Oct 2025 17:50:09 +0300 Subject: [PATCH 4/4] adds more handling on propertyListenerProc --- .../Streaming/Audio Entry/AudioEntry.swift | 3 +++ .../Audio Entry/Models/AudioStreamState.swift | 1 + .../Processors/AudioFileStreamProcessor.swift | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift index 539d848..1ddb08a 100644 --- a/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift +++ b/AudioStreaming/Streaming/Audio Entry/AudioEntry.swift @@ -108,6 +108,9 @@ class AudioEntry { func calculatedBitrate() -> Double { lock.lock(); defer { lock.unlock() } + if let explicitBitRate = audioStreamState.bitRate, explicitBitRate > 0 { + return explicitBitRate + } let packets = processedPacketsState if packetDuration > 0 { let packetsCount = packets.count diff --git a/AudioStreaming/Streaming/Audio Entry/Models/AudioStreamState.swift b/AudioStreaming/Streaming/Audio Entry/Models/AudioStreamState.swift index 88159c9..65b2ae3 100644 --- a/AudioStreaming/Streaming/Audio Entry/Models/AudioStreamState.swift +++ b/AudioStreaming/Streaming/Audio Entry/Models/AudioStreamState.swift @@ -12,4 +12,5 @@ final class AudioStreamState { var dataPacketOffset: UInt64? var dataPacketCount: Double = 0 var streamFormat = AudioStreamBasicDescription() + var bitRate: Double? } diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift index d8b46fd..4f4521a 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioFileStreamProcessor.swift @@ -226,6 +226,8 @@ final class AudioFileStreamProcessor { processDataByteCount(entry: entry, fileStream: fileStream) case kAudioFileStreamProperty_AudioDataPacketCount: processAudioDataPacketCount(entry: entry, fileStream: fileStream) + case kAudioFileStreamProperty_BitRate: + processBitRate(entry: entry, fileStream: fileStream) case kAudioFileStreamProperty_ReadyToProducePackets: // check converter for discontinuous stream assignMagicCookieToConverterIfNeeded() @@ -233,6 +235,8 @@ final class AudioFileStreamProcessor { processReadyToProducePackets(entry: entry, fileStream: fileStream) case kAudioFileStreamProperty_FormatList: processFormatList(entry: entry, fileStream: fileStream) + case kAudioFileStreamProperty_PacketTableInfo: + processPacketTableInfo(entry: entry, fileStream: fileStream) default: break } @@ -336,6 +340,27 @@ final class AudioFileStreamProcessor { entry.audioStreamState.dataPacketOffset = audioDataPacketCount } + private func processBitRate(entry: AudioEntry, fileStream: AudioFileStreamID) { + var bitRate: UInt32 = 0 + let status = fileStreamGetProperty(value: &bitRate, fileStream: fileStream, propertyId: kAudioFileStreamProperty_BitRate) + guard status == noErr else { return } + entry.lock.lock(); defer { entry.lock.unlock() } + entry.audioStreamState.bitRate = Double(bitRate) + } + + private func processPacketTableInfo(entry: AudioEntry, fileStream: AudioFileStreamID) { + var pti = AudioFilePacketTableInfo(mNumberValidFrames: 0, + mPrimingFrames: 0, + mRemainderFrames: 0) + let status = fileStreamGetProperty(value: &pti, fileStream: fileStream, propertyId: kAudioFileStreamProperty_PacketTableInfo) + guard status == noErr else { return } + // Use valid frames to refine duration if present + entry.lock.lock(); defer { entry.lock.unlock() } + if pti.mNumberValidFrames > 0 { + entry.audioStreamState.dataPacketCount = Double(pti.mNumberValidFrames) / Double(max(1, entry.audioStreamFormat.mFramesPerPacket)) + } + } + private func processFormatList(entry: AudioEntry, fileStream: AudioFileStreamID) { let info = fileStreamGetPropertyInfo(fileStream: fileStream, propertyId: kAudioFileStreamProperty_FormatList) guard info.status == noErr, info.size > 0 else { return }