Cherry pick PR #9309: android: Optimize av1 codec initialization

cobalt/testing/filters/android-arm/starboard_platform_tests_filter.json

@@ -1,6 +1,7 @@
 {
   "comment": "TODO: b/450568428 - Investigate these failing tests",
   "failing_tests": [
+    "MediaCapabilitiesCache*",
     "MimeUtilTest.ChecksSupportedFlacContainers",
     "MimeUtilTest.ChecksSupportedMp3Containers",
     "MimeUtilTest.ChecksSupportedPcmContainers",

cobalt/testing/filters/android-arm64/starboard_platform_tests_filter.json

@@ -1,6 +1,7 @@
 {
   "comment": "TODO: b/450568428 - Investigate these failing tests",
   "failing_tests": [
+    "MediaCapabilitiesCache*",
     "MimeUtilTest.ChecksSupportedFlacContainers",
     "MimeUtilTest.ChecksSupportedMp3Containers",
     "MimeUtilTest.ChecksSupportedPcmContainers",

starboard/android/shared/media_capabilities_cache.cc

@@ -397,6 +397,14 @@ bool MediaCapabilitiesCache::IsPassthroughSupported(SbMediaAudioCodec codec) {
   return supported;
 }
 
+bool MediaCapabilitiesCache::IsAv18kCappedAt30() {
+  if (!is_enabled_) {
+    // When the cache is not enabled, always checks video fps.
+    return true;
+  }
+  return is_av1_8k_capped_at_30_;
+}
+
 bool MediaCapabilitiesCache::GetAudioConfiguration(
     int index,
     SbMediaAudioConfiguration* configuration) {
@@ -562,6 +570,7 @@ void MediaCapabilitiesCache::UpdateMediaCapabilities_Locked() {
   media_capabilities_provider_->GetCodecCapabilities(
       audio_codec_capabilities_map_, video_codec_capabilities_map_);
   LoadAudioConfigurations_Locked();
+  LoadIsAv18kCappedAt30_Locked();
 }
 
 void MediaCapabilitiesCache::LoadAudioConfigurations_Locked() {
@@ -579,4 +588,28 @@ void MediaCapabilitiesCache::LoadAudioConfigurations_Locked() {
   }
 }
 
+void MediaCapabilitiesCache::LoadIsAv18kCappedAt30_Locked() {
+  const bool enable_av1_startup_optimization =
+      starboard::features::FeatureList::IsEnabled(
+          starboard::features::kEnableAv1StartupOptimization);
+  if (!enable_av1_startup_optimization) {
+    return;
+  }
+
+  is_av1_8k_capped_at_30_ = false;
+  for (const auto& video_capability :
+       video_codec_capabilities_map_[SupportedVideoCodecToMimeType(
+           kSbMediaVideoCodecAv1)]) {
+    constexpr int kWidth8K = 7680;
+    constexpr int kHeight8K = 4320;
+    if (video_capability->AreResolutionAndRateSupported(kWidth8K, kHeight8K,
+                                                        /*fps=*/0) &&
+        !video_capability->AreResolutionAndRateSupported(kWidth8K, kHeight8K,
+                                                         /*fps=*/60)) {
+      is_av1_8k_capped_at_30_ = true;
+      break;
+    }
+  }
+}
+
 }  // namespace starboard

starboard/android/shared/media_capabilities_cache.h

@@ -177,6 +177,13 @@ class MediaCapabilitiesCache {
 
   bool IsPassthroughSupported(SbMediaAudioCodec codec);
 
+  // Some android devices support av1 up to 8k30 and 4k60. In that case, we
+  // cannot ask it to always use max supported width and height, which would
+  // lead 8k60 being used and exceed system resource limit. See b/173575800.
+  // When IsAv18kCappedAt30() returns true, the device needs extra check of
+  // video fps to prevent 8k60 is used unexpectedly.
+  bool IsAv18kCappedAt30();
+
   bool GetAudioConfiguration(int index,
                              SbMediaAudioConfiguration* configuration);
 
@@ -219,6 +226,7 @@ class MediaCapabilitiesCache {
 
   void UpdateMediaCapabilities_Locked();
   void LoadAudioConfigurations_Locked();
+  void LoadIsAv18kCappedAt30_Locked();
 
   std::mutex mutex_;
 
@@ -238,6 +246,7 @@ class MediaCapabilitiesCache {
   std::vector<SbMediaAudioConfiguration> audio_configurations_;
   bool is_widevine_supported_ = false;
   bool is_cbcs_supported_ = false;
+  bool is_av1_8k_capped_at_30_ = true;
 
   std::atomic_bool is_enabled_{true};
   std::atomic_bool capabilities_is_dirty_{true};

starboard/android/shared/video_decoder.cc

@@ -26,6 +26,7 @@
 
 #include "base/android/jni_android.h"
 #include "build/build_config.h"
+#include "starboard/android/shared/media_capabilities_cache.h"
 #include "starboard/android/shared/media_common.h"
 #include "starboard/android/shared/video_render_algorithm.h"
 #include "starboard/android/shared/video_surface_texture_bridge.h"
@@ -416,6 +417,9 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
       flush_delay_usec_(android_get_device_api_level() < 34 ? flush_delay_usec
                                                             : 0),
       force_reset_surface_(force_reset_surface),
+      needs_fps_to_initialize_codec_(
+          video_codec_ == kSbMediaVideoCodecAv1 &&
+          MediaCapabilitiesCache::GetInstance()->IsAv18kCappedAt30()),
       is_video_frame_tracker_enabled_(IsFrameRenderedCallbackEnabled() ||
                                       tunnel_mode_audio_session_id != -1),
       has_new_texture_available_(false),
@@ -447,7 +451,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
     SB_DCHECK_EQ(output_mode_, kSbPlayerOutputModeDecodeToTexture);
   }
 
-  if (video_codec_ != kSbMediaVideoCodecAv1) {
+  if (!needs_fps_to_initialize_codec_) {
     auto result = InitializeCodec(video_stream_info);
     if (!result) {
       *error_message =
@@ -560,7 +564,7 @@ void MediaCodecVideoDecoder::WriteInputBuffers(
 
     // Re-initialize the codec now if it was torn down either in |Reset| or
     // because we need to change the color metadata.
-    if (video_codec_ != kSbMediaVideoCodecAv1 && media_decoder_ == NULL) {
+    if (!needs_fps_to_initialize_codec_ && media_decoder_ == NULL) {
       auto result = InitializeCodec(input_buffers.front()->video_stream_info());
       if (!result) {
         std::string error_message =
@@ -581,7 +585,7 @@ void MediaCodecVideoDecoder::WriteInputBuffers(
 
   input_buffer_written_ += input_buffers.size();
 
-  if (video_codec_ == kSbMediaVideoCodecAv1 && video_fps_ == 0) {
+  if (needs_fps_to_initialize_codec_ && video_fps_ == 0) {
     SB_DCHECK(!media_decoder_);
 
     pending_input_buffers_.insert(pending_input_buffers_.end(),
@@ -625,7 +629,7 @@ void MediaCodecVideoDecoder::WriteEndOfStream() {
     return;
   }
 
-  if (video_codec_ == kSbMediaVideoCodecAv1 && video_fps_ == 0) {
+  if (needs_fps_to_initialize_codec_ && video_fps_ == 0) {
     SB_DCHECK(!media_decoder_);
     SB_DCHECK_EQ(pending_input_buffers_.size(),
                  static_cast<size_t>(input_buffer_written_));
@@ -660,7 +664,7 @@ void MediaCodecVideoDecoder::Reset() {
   SB_CHECK(BelongsToCurrentThread());
 
   // If fail to flush |media_decoder_| or |media_decoder_| is null, then
-  // re-create |media_decoder_|. If the codec is kSbMediaVideoCodecAv1,
+  // re-create |media_decoder_|. If |needs_fps_to_initialize_codec_| is true,
   // set video_fps_ to 0 will call InitializeCodec(),
   // which we do not need if flush the codec.
   if (!enable_flush_during_seek_ || !media_decoder_ ||
@@ -701,7 +705,7 @@ Result<void> MediaCodecVideoDecoder::InitializeCodec(
     const VideoStreamInfo& video_stream_info) {
   SB_CHECK(BelongsToCurrentThread());
 
-  if (video_stream_info.codec == kSbMediaVideoCodecAv1) {
+  if (needs_fps_to_initialize_codec_) {
     SB_DCHECK_GT(pending_input_buffers_.size(), 0u);
 
     // Guesstimate the video fps.
@@ -779,7 +783,7 @@ Result<void> MediaCodecVideoDecoder::InitializeCodec(
     return Failure("Video surface does not exist.");
   }
 
-  if (video_stream_info.codec == kSbMediaVideoCodecAv1) {
+  if (needs_fps_to_initialize_codec_) {
     SB_DCHECK_GT(video_fps_, 0);
   } else {
     SB_DCHECK_EQ(video_fps_, 0);
@@ -809,7 +813,7 @@ Result<void> MediaCodecVideoDecoder::InitializeCodec(
     }
     media_decoder_->SetPlaybackRate(playback_rate_);
 
-    if (video_stream_info.codec == kSbMediaVideoCodecAv1) {
+    if (needs_fps_to_initialize_codec_) {
       SB_DCHECK(!pending_input_buffers_.empty());
     } else {
       SB_DCHECK(pending_input_buffers_.empty());

starboard/android/shared/video_decoder.h

@@ -192,6 +192,10 @@ class MediaCodecVideoDecoder : public VideoDecoder,
   // Force resetting the video surface after every playback.
   const bool force_reset_surface_;
 
+  // Codec initialization will be delayed until the decoder receives enough
+  // inputs to estimate video fps when |needs_fps_to_initialize_codec_| is true.
+  const bool needs_fps_to_initialize_codec_;
+
   // On some platforms tunnel mode is only supported in the secure pipeline.  So
   // we create a dummy drm system to force the video playing in secure pipeline
   // to enable tunnel mode.

starboard/extension/feature_config.h

@@ -161,6 +161,11 @@ STARBOARD_FEATURE(kVideoDecoderDelayUsecOverride,
 STARBOARD_FEATURE(kRejectLowPerformanceSoftwareDecoder,
                   "RejectLowPerformanceSoftwareDecoder",
                   false)
+
+// Set the following variable to true to enable av1 startup optimization.
+STARBOARD_FEATURE(kEnableAv1StartupOptimization,
+                  "EnableAv1StartupOptimization",
+                  false)
 #endif  // BUILDFLAG(IS_ANDROID) && (SB_API_VERSION >= 17)
 FEATURE_LIST_END