diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java index cccf8c61dff..a3dd1579796 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java @@ -217,6 +217,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Random; @@ -259,6 +260,9 @@ public final class ExoPlayerTest { private static final int TIMEOUT_MS = 10_000; private static final String SAMPLE_URI = "asset://android_asset/media/mp4/sample.mp4"; + private static final String SAMPLE_AC4_TS_URI = + "asset://android_asset/media/ts/" + + "MultiLangPerso_1PID_PC0_Select_AC4_H265_DVB_50fps_Audio_Only.ts"; @Parameters(name = "preload={0}") public static ImmutableList params() { @@ -14414,6 +14418,72 @@ public void playingMedia_withNoMetadata_doesNotUpdateMediaMetadata() throws Exce player.release(); } + @SuppressWarnings("UseSdkSuppress") // https://issuetracker.google.com/382253664 + @RequiresApi(api = Build.VERSION_CODES.P) + @Test + public void playingAC4_TS_withMultipleAudioPresentations() throws Exception { + ExoPlayer player = parameterizeTestExoPlayerBuilder(new TestExoPlayerBuilder(context)).build(); + player.setMediaItem(MediaItem.fromUri(SAMPLE_AC4_TS_URI)); + List refPresentations = new ArrayList<>(); + HashMap labelsFirst = new HashMap<>(); + labelsFirst.put(new ULocale("en"), "Standard"); + HashMap labelsSecond = new HashMap<>(); + labelsSecond.put(new ULocale("en"), "Kids' choice"); + HashMap labelsThird = new HashMap<>(); + labelsThird.put(new ULocale("en"), "Artists' commentary"); + refPresentations.add(new AudioPresentation.Builder(10) + .setLocale(ULocale.ENGLISH) + .setMasteringIndication(AudioPresentation.MASTERED_FOR_SURROUND) + .setHasSpokenSubtitles(false) + .setLabels(labelsFirst) + .setHasDialogueEnhancement(true) + .build()); + refPresentations.add(new AudioPresentation.Builder(11) + .setLocale(ULocale.ENGLISH) + .setMasteringIndication(AudioPresentation.MASTERED_FOR_SURROUND) + .setHasSpokenSubtitles(false) + .setLabels(labelsSecond) + .setHasAudioDescription(true) + .setHasDialogueEnhancement(true) + .build()); + refPresentations.add(new AudioPresentation.Builder(12) + .setLocale(ULocale.FRENCH) + .setMasteringIndication(AudioPresentation.MASTERED_FOR_SURROUND) + .setHasSpokenSubtitles(false) + .setLabels(labelsThird) + .setHasAudioDescription(false) + .setHasDialogueEnhancement(true) + .build()); + final boolean[] audioPresentationsVerfied = {false}; + player.addListener( + new Player.Listener() { + @Override + public void onAudioPresentationsChanged(List audioPresentations) { + + assertThat(audioPresentations).isNotNull(); + assertThat(audioPresentations.size()).isEqualTo(refPresentations.size()); + for (int i = 0; i < audioPresentations.size(); i++) { + assertThat(audioPresentations.get(i)).isEqualTo((refPresentations.get(i))); + } + audioPresentationsVerfied[0] = true; + } + }); + + player.prepare(); + player.play(); + try { + runUntilPlaybackState(player, Player.STATE_ENDED); + } catch (IllegalStateException ignored) { + // Not expected to play the input. + } + assertThat(audioPresentationsVerfied[0]).isTrue(); + player.stop(); + + shadowOf(Looper.getMainLooper()).idle(); + + player.release(); + } + @Test @Config(sdk = ALL_SDKS) public void builder_inBackgroundThreadWithAllowedAnyThreadMethods_doesNotThrow() diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/Ac4Reader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/Ac4Reader.java index 0d97e8a6e72..a502d09d7c0 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/Ac4Reader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/Ac4Reader.java @@ -19,6 +19,7 @@ import static java.lang.Math.min; import static java.lang.annotation.ElementType.TYPE_USE; +import android.media.AudioPresentation; import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.media3.common.C; @@ -33,10 +34,12 @@ import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.TrackOutput; import androidx.media3.extractor.ts.TsPayloadReader.TrackIdGenerator; +import com.google.common.collect.ImmutableList; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -59,6 +62,7 @@ public final class Ac4Reader implements ElementaryStreamReader { @Nullable private final String language; private final @C.RoleFlags int roleFlags; private final String containerMimeType; + private List audioPresentations; private @MonotonicNonNull String formatId; private @MonotonicNonNull TrackOutput output; @@ -106,6 +110,11 @@ public Ac4Reader( this.language = language; this.roleFlags = roleFlags; this.containerMimeType = containerMimeType; + audioPresentations = ImmutableList.of(); + } + + public void setAudioPresentations(List audioPresentations) { + this.audioPresentations = ImmutableList.copyOf(audioPresentations); } @Override @@ -220,7 +229,8 @@ private void parseHeader() { if (format == null || frameInfo.channelCount != format.channelCount || frameInfo.sampleRate != format.sampleRate - || !MimeTypes.AUDIO_AC4.equals(format.sampleMimeType)) { + || !MimeTypes.AUDIO_AC4.equals(format.sampleMimeType) + || !format.audioPresentations.equals(audioPresentations)) { format = new Format.Builder() .setId(formatId) @@ -230,6 +240,7 @@ private void parseHeader() { .setSampleRate(frameInfo.sampleRate) .setLanguage(language) .setRoleFlags(roleFlags) + .setAudioPresentations(audioPresentations) .build(); output.format(format); } diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/PesReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/PesReader.java index d1dbdc98298..7d5763cc032 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/PesReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/PesReader.java @@ -17,6 +17,7 @@ import static java.lang.Math.min; +import android.media.AudioPresentation; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ParserException; @@ -27,6 +28,7 @@ import androidx.media3.common.util.TimestampAdjuster; import androidx.media3.common.util.UnstableApi; import androidx.media3.extractor.ExtractorOutput; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -181,6 +183,17 @@ public boolean canConsumeSynthesizedEmptyPusi(boolean isModeHls) { && headerParsed; } + /** + * Sets the audio presentation available for current AC-4 audio track. + * + * @param audioPresentations This is a collection of {@link AudioPresentation}. + */ + public void setAudioPresentations(List audioPresentations) { + if (reader instanceof Ac4Reader) { + ((Ac4Reader) reader).setAudioPresentations(audioPresentations); + } + } + private void setState(int state) { this.state = state; bytesRead = 0; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/TsExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/TsExtractor.java index 73b75e1ac1d..b4aa1eb66f5 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/TsExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/TsExtractor.java @@ -19,11 +19,16 @@ import static androidx.media3.extractor.ts.TsPayloadReader.FLAG_PAYLOAD_UNIT_START_INDICATOR; import static java.lang.annotation.ElementType.TYPE_USE; +import android.icu.util.ULocale; +import android.media.AudioPresentation; +import android.os.Build; +import android.util.Log; import android.util.SparseArray; import android.util.SparseBooleanArray; import android.util.SparseIntArray; import androidx.annotation.IntDef; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.ParserException; @@ -54,13 +59,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.ListIterator; +import java.util.Locale; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** Extracts data from the MPEG-2 TS container format. */ @UnstableApi public final class TsExtractor implements Extractor { + private static final String TAG = "TsExtractor"; /** * Creates a factory for {@link TsExtractor} instances with the provided {@link * SubtitleParser.Factory}. @@ -149,6 +158,10 @@ public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParser public static final int TS_SYNC_BYTE = 0x47; // First byte of each TS packet. private static final int TS_PAT_PID = 0; + private static final int TS_TABLE_ID_SERVICE_DESCRIPTION_SECTION = 0x42; + private static final int TS_SERVICE_DESCRIPTOR_TAG = 0x48; + // Service Description Table, Stuffing Table and Bouquet Association Table all have the same PID. + private static final int TS_SDT_BAT_ST_PID = 0x11; private static final int MAX_PID_PLUS_ONE = 0x2000; private static final long AC3_FORMAT_IDENTIFIER = 0x41432d33; @@ -170,10 +183,15 @@ public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParser private final SparseArray tsPayloadReaders; // Indexed by pid private final SparseBooleanArray trackIds; private final SparseBooleanArray trackPids; + + private final SparseIntArray presentationMessageIds; private final TsDurationReader durationReader; // Accessed only by the loading thread. private @MonotonicNonNull TsBinarySearchSeeker tsBinarySearchSeeker; + + List audioPresentations; + boolean pendingAudioPresentationsLabels; private ExtractorOutput output; private int remainingPmts; private boolean tracksEnded; @@ -340,6 +358,9 @@ public TsExtractor( tsPacketBuffer = new ParsableByteArray(new byte[BUFFER_SIZE], 0); trackIds = new SparseBooleanArray(); trackPids = new SparseBooleanArray(); + presentationMessageIds = new SparseIntArray(); + audioPresentations = new ArrayList<>(); + pendingAudioPresentationsLabels = false; tsPayloadReaders = new SparseArray<>(); continuityCounters = new SparseIntArray(); durationReader = new TsDurationReader(timestampSearchBytes); @@ -520,6 +541,10 @@ public void release() { boolean wereTracksEnded = tracksEnded; if (shouldConsumePacketPayload(pid)) { tsPacketBuffer.setLimit(endOfPacket); + if (payloadReader instanceof PesReader && + (!pendingAudioPresentationsLabels)) { + ((PesReader) payloadReader).setAudioPresentations(audioPresentations); + } payloadReader.consume(tsPacketBuffer, packetHeaderFlags); tsPacketBuffer.setLimit(limit); } @@ -620,7 +645,41 @@ private void resetPayloadReaders() { tsPayloadReaders.put(initialPayloadReaders.keyAt(i), initialPayloadReaders.valueAt(i)); } tsPayloadReaders.put(TS_PAT_PID, new SectionReader(new PatReader())); + tsPayloadReaders.put(TS_SDT_BAT_ST_PID, new SectionReader(new SdtSectionReader())); id3Reader = null; + audioPresentations.clear(); + presentationMessageIds.clear(); + pendingAudioPresentationsLabels = false; + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private void updatePresentationLabels(List aps, + SparseArray> presentationLabels) { + for (ListIterator itr = aps.listIterator(); itr.hasNext(); ) { + AudioPresentation ap = itr.next(); + final int valueIfKeyNotFound = -1; + final int messageId = presentationMessageIds.get(ap.getPresentationId(), valueIfKeyNotFound); + if (messageId == valueIfKeyNotFound) { + Log.e(TAG, "No message id detected"); + return; + } + final HashMap labels = presentationLabels.get(messageId); + if (labels.isEmpty()) { + continue; + } + // Reconstruct the AudioPresentation object with valid labels in it. + AudioPresentation newAp = (new AudioPresentation.Builder(ap.getPresentationId()) + .setProgramId(ap.getProgramId()) + .setLocale(ULocale.forLocale(ap.getLocale())) + .setLabels(labels) + .setMasteringIndication(ap.getMasteringIndication()) + .setHasAudioDescription(ap.hasAudioDescription()) + .setHasSpokenSubtitles(ap.hasSpokenSubtitles()) + .setHasDialogueEnhancement(ap.hasDialogueEnhancement())).build(); + itr.set(newAp); + presentationMessageIds.delete(ap.getPresentationId()); + } + pendingAudioPresentationsLabels = false; } /** Parses Program Association Table data. */ @@ -678,6 +737,103 @@ public void consume(ParsableByteArray sectionData) { } } + /** + * Parses Service Description Table data. + */ + private class SdtSectionReader implements SectionPayloadReader { + public SdtSectionReader() { + } + @Override + public void init(TimestampAdjuster timestampAdjuster, ExtractorOutput extractorOutput, + TrackIdGenerator idGenerator) { + // Do nothing. + } + @RequiresApi(api = Build.VERSION_CODES.R) + @Override + public void consume(ParsableByteArray sectionData) { + if (sectionData.bytesLeft() < 1) { + return; + } + final int tableId = sectionData.readUnsignedByte(); + if (tableId != TS_TABLE_ID_SERVICE_DESCRIPTION_SECTION) { + // See DVB BlueBook A038, section 5.1.3 for more information on table id assignment. + return; + } + if (sectionData.bytesLeft() < 15) { + return; + } + // section_syntax_indicator(1), reserved_future_use(1), reserved(2), + // section_length(12), transport_stream_id(16), reserved(2), version_number(5), + // current_next_indicator(1), section_number(8), last_section_number(8), + // original_network_id(16), reserved_future_use(8) + sectionData.skipBytes(10); + sectionData.skipBytes(2); // serviceId + // reserved_future_use(6), EIT_schedule_flag(1), EIT_present_following_flag(1) + sectionData.skipBytes(1); + // Remove running_status(3), free_CA_mode(1) from the descriptors_loop_length with the mask. + final int descriptorsLoopLength = sectionData.readUnsignedShort() & 0xFFF; + if (descriptorsLoopLength != sectionData.bytesLeft()) { + Log.e(TAG, "Invalid section data length"); + return; + } + // Indexed by message id and value contains labels in each language + SparseArray> presentationLabels = new SparseArray<>(); + while (sectionData.bytesLeft() > 0) { + final int descriptorTag = sectionData.readUnsignedByte(); + if (sectionData.bytesLeft() < 1) { + break; + } + int descriptorLength = sectionData.readUnsignedByte(); + if (sectionData.bytesLeft() < descriptorLength) { + break; + } + if (descriptorTag == PmtReader.TS_PMT_DESC_DVB_EXT) { + final int descriptorTagExt = sectionData.readUnsignedByte(); + int descriptorExtLength = descriptorLength - 1; + // Message_descriptor in DVB A038 + if (descriptorTagExt == PmtReader.TS_PMT_DESC_DVB_EXT_MESSAGE) { + int message_id = sectionData.readUnsignedByte(); + descriptorExtLength--; + if (sectionData.bytesLeft() < 3) { + break; + } + String language = new String(sectionData.getData(), + sectionData.getPosition(), 3).trim(); + sectionData.setPosition(sectionData.getPosition() + 3); + descriptorExtLength -= 3; + if (descriptorExtLength < 1) { + break; + } + CharSequence label = new String( + sectionData.getData(), sectionData.getPosition(), descriptorExtLength).trim(); + sectionData.setPosition(sectionData.getPosition() + descriptorExtLength); + HashMap labels = new HashMap() + {{ put(ULocale.forLocale(new Locale(language)), label); }}; + presentationLabels.append(message_id, labels); + } else { + // Ignore other extended descriptors. + sectionData.skipBytes(descriptorExtLength - 1); + } + } else if (descriptorTag == TS_SERVICE_DESCRIPTOR_TAG) { + final int serviceType = sectionData.readUnsignedByte(); + Log.d(TAG,"serviceType : " + serviceType); + final int serviceProviderNameLength = sectionData.readUnsignedByte(); + String serviceProvider = sectionData.readString(serviceProviderNameLength); + Log.d(TAG,"serviceProvider : " + serviceProvider); + final int serviceNameLength = sectionData.readUnsignedByte(); + String serviceName = sectionData.readString(serviceNameLength); + Log.d(TAG,"serviceName: " + serviceName); + } else { + sectionData.skipBytes(descriptorLength); + } + } + updatePresentationLabels(audioPresentations, presentationLabels); + if (mode != MODE_HLS) { + tsPayloadReaders.remove(TS_SDT_BAT_ST_PID); + } + } + } + /** Parses Program Map Table. */ private class PmtReader implements SectionPayloadReader { @@ -688,10 +844,12 @@ private class PmtReader implements SectionPayloadReader { private static final int TS_PMT_DESC_EAC3 = 0x7A; private static final int TS_PMT_DESC_DTS = 0x7B; private static final int TS_PMT_DESC_DVB_EXT = 0x7F; + private static final int TS_PMT_DESC_DVB_EXT_MESSAGE = 0x08; private static final int TS_PMT_DESC_DVBSUBS = 0x59; private static final int TS_PMT_DESC_DVB_EXT_AC4 = 0x15; private static final int TS_PMT_DESC_DVB_EXT_DTS_HD = 0x0E; + private static final int TS_PMT_DESC_DVB_EXT_AUDIO_PRESELECTION = 0x19; private static final int TS_PMT_DESC_DVB_EXT_DTS_UHD = 0x21; private final ParsableBitArray pmtScratch; @@ -763,7 +921,8 @@ public void consume(ParsableByteArray sectionData) { // Setup an ID3 track regardless of whether there's a corresponding entry, in case one // appears intermittently during playback. See [Internal: b/20261500]. EsInfo id3EsInfo = - new EsInfo(TS_STREAM_TYPE_ID3, null, AUDIO_TYPE_UNDEFINED, null, Util.EMPTY_BYTE_ARRAY); + new EsInfo(TS_STREAM_TYPE_ID3, null, AUDIO_TYPE_UNDEFINED, null, + Util.EMPTY_BYTE_ARRAY); id3Reader = payloadReaderFactory.createPayloadReader(TS_STREAM_TYPE_ID3, id3EsInfo); if (id3Reader != null) { id3Reader.init( @@ -881,9 +1040,94 @@ private EsInfo readEsInfo(ParsableByteArray data, int length) { } else if (descriptorTag == TS_PMT_DESC_DVB_EXT) { // Extension descriptor in DVB (ETSI EN 300 468). int descriptorTagExt = data.readUnsignedByte(); + final int descriptorExtLength = descriptorLength - 1; if (descriptorTagExt == TS_PMT_DESC_DVB_EXT_AC4) { // AC-4_descriptor in DVB (ETSI EN 300 468). streamType = TS_STREAM_TYPE_AC4; + } else if (descriptorTagExt == TS_PMT_DESC_DVB_EXT_AUDIO_PRESELECTION) { + // Audio_preselection_descriptor in DVB A038 + ParsableBitArray extData = new ParsableBitArray(new byte[descriptorExtLength]); + data.readBytes(extData, descriptorExtLength); + if (extData.bitsLeft() < 8) { + break; + } + int numPreselections = extData.readBits(5); + extData.skipBits(3); // reserved_zero_future_use + audioPresentations.clear(); + pendingAudioPresentationsLabels = false; + for (int i = 0; i < numPreselections; i++) { + if (extData.bitsLeft() < ((numPreselections - i) * 16)) { + break; + } + int presentationId = extData.readBits(5); + int masteringIndication = extData.readBits(3); // audio_rendering_indication + boolean audioDescriptionAvailable = extData.readBit(); + boolean spokenSubtitlesAvailable = extData.readBit(); + boolean dialogueEnhancementAvailable = extData.readBit(); + extData.skipBits(1); // interactivityEnabled + boolean languageCodePresent = extData.readBit(); + boolean textLabelPresent = extData.readBit(); + boolean multiStreamInfoPresent = extData.readBit(); + boolean futureExtension = extData.readBit(); + String languageTag = null; + if (languageCodePresent) { + if (extData.bitsLeft() < 24) { + break; + } + languageTag = new String(extData.data, extData.getPosition()/8, 3).trim(); + extData.setPosition(extData.getPosition() + 24); + } + if (textLabelPresent) { + if (extData.bitsLeft() < 8) { + break; + } + int messageId = extData.readBits(8); + presentationMessageIds.put(presentationId, messageId); + pendingAudioPresentationsLabels = true; + } + if (multiStreamInfoPresent) { + if (extData.bitsLeft() < 8) { + break; + } + int numAuxComponents = extData.readBits(3); + extData.skipBits(5); // reserved_zero_future_use + if (extData.bitsLeft() < (numAuxComponents * 8)) { + break; + } + extData.skipBits(numAuxComponents * 8); // component_tag + } + if (futureExtension) { + if (extData.bitsLeft() < 8) { + break; + } + extData.skipBits(3); // reserved + int futureExtensionLength = extData.readBits(5); + if (extData.bitsLeft() < (futureExtensionLength * 8)) { + break; + } + extData.skipBits(futureExtensionLength * 8); // futureExtensionByte + } + if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + AudioPresentation.Builder presentation = + (new AudioPresentation.Builder(presentationId) + .setMasteringIndication(masteringIndication) + .setHasAudioDescription(audioDescriptionAvailable) + .setHasSpokenSubtitles(spokenSubtitlesAvailable) + .setHasDialogueEnhancement(dialogueEnhancementAvailable)); + if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { + presentation.setProgramId(AudioPresentation.PROGRAM_ID_UNKNOWN); + } else { + presentation.setProgramId(-1); + } + if (languageTag != null) { + presentation.setLocale(ULocale.forLocale(new Locale(languageTag))); + } + audioPresentations.add(presentation.build()); + } + } + if (!pendingAudioPresentationsLabels) { + Log.d(TAG,"Audio presentations without labels: " + audioPresentations); + } } else if (descriptorTagExt == TS_PMT_DESC_DVB_EXT_DTS_HD) { // DTS-HD descriptor in DVB (ETSI EN 300 468). streamType = TS_STREAM_TYPE_DTS_HD; diff --git a/libraries/test_data/src/test/assets/media/ts/MultiLangPerso_1PID_PC0_Select_AC4_H265_DVB_50fps_Audio_Only.ts b/libraries/test_data/src/test/assets/media/ts/MultiLangPerso_1PID_PC0_Select_AC4_H265_DVB_50fps_Audio_Only.ts new file mode 100644 index 00000000000..b20f84e9ee6 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/ts/MultiLangPerso_1PID_PC0_Select_AC4_H265_DVB_50fps_Audio_Only.ts differ