H.265 for Apple platforms

api/crypto/frame_crypto_transformer.cc

@@ -32,6 +32,7 @@
 #include "absl/types/variant.h"
 #include "api/array_view.h"
 #include "common_video/h264/h264_common.h"
+#include "common_video/h265/h265_common.h"
 #include "modules/rtp_rtcp/source/rtp_format_h264.h"
 #include "rtc_base/byte_buffer.h"
 #include "rtc_base/logging.h"
@@ -96,6 +97,40 @@ inline bool FrameIsH264(webrtc::TransformableFrameInterface* frame,
   }
 }
 
+inline bool FrameIsH265(webrtc::TransformableFrameInterface* frame,
+                        webrtc::FrameCryptorTransformer::MediaType type) {
+  switch (type) {
+    case webrtc::FrameCryptorTransformer::MediaType::kVideoFrame: {
+      auto videoFrame =
+          static_cast<webrtc::TransformableVideoFrameInterface*>(frame);
+      return videoFrame->header().codec ==
+             webrtc::VideoCodecType::kVideoCodecH265;
+    }
+    default:
+      return false;
+  }
+}
+
+inline bool IsH265SliceNalu(webrtc::H265::NaluType nalu_type) {
+  // VCL NALUs (Video Coding Layer) - slice segments
+  return nalu_type == webrtc::H265::NaluType::kTrailN ||
+         nalu_type == webrtc::H265::NaluType::kTrailR ||
+         nalu_type == webrtc::H265::NaluType::kTsaN ||
+         nalu_type == webrtc::H265::NaluType::kTsaR ||
+         nalu_type == webrtc::H265::NaluType::kStsaN ||
+         nalu_type == webrtc::H265::NaluType::kStsaR ||
+         nalu_type == webrtc::H265::NaluType::kRadlN ||
+         nalu_type == webrtc::H265::NaluType::kRadlR ||
+         nalu_type == webrtc::H265::NaluType::kRaslN ||
+         nalu_type == webrtc::H265::NaluType::kRaslR ||
+         nalu_type == webrtc::H265::NaluType::kBlaWLp ||
+         nalu_type == webrtc::H265::NaluType::kBlaWRadl ||
+         nalu_type == webrtc::H265::NaluType::kBlaNLp ||
+         nalu_type == webrtc::H265::NaluType::kIdrWRadl ||
+         nalu_type == webrtc::H265::NaluType::kIdrNLp ||
+         nalu_type == webrtc::H265::NaluType::kCra;
+}
+
 inline bool NeedsRbspUnescaping(const uint8_t* frameData, size_t frameSize) {
   for (size_t i = 0; i < frameSize - 3; ++i) {
     if (frameData[i] == 0 && frameData[i + 1] == 0 && frameData[i + 2] == 3)
@@ -163,6 +198,27 @@ uint8_t get_unencrypted_bytes(webrtc::TransformableFrameInterface* frame,
               break;
           }
         }
+      } else if (videoFrame->header().codec ==
+                 webrtc::VideoCodecType::kVideoCodecH265) {
+        rtc::ArrayView<const uint8_t> data_in = frame->GetData();
+        std::vector<webrtc::H265::NaluIndex> nalu_indices =
+            webrtc::H265::FindNaluIndices(data_in);
+
+        int idx = 0;
+        for (const auto& index : nalu_indices) {
+          const uint8_t* slice = data_in.data() + index.payload_start_offset;
+          webrtc::H265::NaluType nalu_type =
+              webrtc::H265::ParseNaluType(slice[0]);
+          if (IsH265SliceNalu(nalu_type)) {
+            // H.265 has a 2-byte NALU header, so unencrypted bytes = offset + header size
+            unencrypted_bytes = index.payload_start_offset + webrtc::H265::kNaluHeaderSize;
+            RTC_LOG(LS_INFO)
+                << "H265 NonParameterSetNalu::payload_size: " << index.payload_size
+                << ", nalu_type " << static_cast<int>(nalu_type) << ", NaluIndex [" << idx++
+                << "] offset: " << index.payload_start_offset << ", unencrypted_bytes: " << unencrypted_bytes;
+            return unencrypted_bytes;
+          }
+        }
       }
       break;
     }
@@ -413,6 +469,9 @@ void FrameCryptorTransformer::encryptFrame(
     if (FrameIsH264(frame.get(), type_)) {
       H264::WriteRbsp(data_without_header.data(), data_without_header.size(),
                       &data_out);
+    } else if (FrameIsH265(frame.get(), type_)) {
+      H265::WriteRbsp(data_without_header.data(), data_without_header.size(),
+                      &data_out);
     } else {
       data_out.AppendData(data_without_header);
       RTC_CHECK_EQ(data_out.size(), frame_header.size() +
@@ -561,6 +620,10 @@ void FrameCryptorTransformer::decryptFrame(
       NeedsRbspUnescaping(encrypted_buffer.data(), encrypted_buffer.size())) {
     encrypted_buffer.SetData(
         H264::ParseRbsp(encrypted_buffer.data(), encrypted_buffer.size()));
+  } else if (FrameIsH265(frame.get(), type_) &&
+             NeedsRbspUnescaping(encrypted_buffer.data(), encrypted_buffer.size())) {
+    encrypted_buffer.SetData(
+        H265::ParseRbsp(encrypted_buffer.data(), encrypted_buffer.size()));
   }
 
   rtc::Buffer encrypted_payload(encrypted_buffer.size() - ivLength - 2);

common_video/h265/h265_common.h

@@ -43,6 +43,8 @@ enum NaluType : uint8_t {
   kStsaR = 5,
   kRadlN = 6,
   kRadlR = 7,
+  kRaslN = 8,
+  kRaslR = 9,
   kBlaWLp = 16,
   kBlaWRadl = 17,
   kBlaNLp = 18,

sdk/BUILD.gn

@@ -759,6 +759,16 @@ if (is_ios || is_mac) {
         ]
       }
 
+      if (rtc_use_h265) {
+        sources += [
+          "objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h",
+          "objc/components/video_codec/RTCCodecSpecificInfoH265.h",
+          "objc/components/video_codec/RTCCodecSpecificInfoH265.mm",
+          "objc/components/video_codec/RTCH265ProfileLevelId.h",
+          "objc/components/video_codec/RTCH265ProfileLevelId.mm",
+        ]
+      }
+
       public_configs = [ ":common_config_objc" ]
       deps = [
         ":base_objc",
@@ -1516,6 +1526,15 @@ if (is_ios || is_mac) {
           "objc/components/audio/RTCAudioProcessingConfig.h",
         ]
 
+        if (rtc_use_h265) {
+          common_objc_headers += [
+            "objc/components/video_codec/RTCCodecSpecificInfoH265.h",
+            "objc/components/video_codec/RTCH265ProfileLevelId.h",
+            "objc/components/video_codec/RTCVideoDecoderH265.h",
+            "objc/components/video_codec/RTCVideoEncoderH265.h",
+          ]
+        }
+
         if (target_environment != "xrdevice" && target_environment != "xrsimulator") {
           common_objc_headers += [
             "objc/helpers/RTCCameraPreviewView.h",
@@ -1919,6 +1938,17 @@ if (is_ios || is_mac) {
         "objc/components/video_codec/RTCVideoEncoderH264.mm",
       ]
 
+      if (rtc_use_h265) {
+        sources += [
+          "objc/components/video_codec/RTCVideoDecoderH265.h",
+          "objc/components/video_codec/RTCVideoDecoderH265.mm",
+          "objc/components/video_codec/RTCVideoEncoderH265.h",
+          "objc/components/video_codec/RTCVideoEncoderH265.mm",
+          "objc/components/video_codec/RTCVideoFrameReorderQueue.h",
+          "objc/components/video_codec/RTCVideoFrameReorderQueue.mm",
+        ]
+      }
+
       configs += [
         "..:common_objc",
         ":used_from_extension",

sdk/objc/components/capturer/RTCCameraVideoCapturer.m

@@ -303,7 +303,7 @@ - (void)stopCaptureWithCompletionHandler:
 #if TARGET_WATCH_DEVICE_ROTATION
 - (void)deviceOrientationDidChange:(NSNotification *)notification {
   [RTC_OBJC_TYPE(RTCDispatcher)
-      dispatchAsyncOnType:RTC_OBJC_TYPE(RTCDispatcherTypeCaptureSession)
+      dispatchAsyncOnType:RTC_OBJC_TYPE(RTCDispatcherTypeMain)
                     block:^{
                       [self updateOrientation];
                     }];

sdk/objc/components/video_codec/RTCCodecSpecificInfoH265+Private.h

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264+Private.h */
+
+#import "RTCCodecSpecificInfoH265.h"
+
+#include "modules/video_coding/include/video_codec_interface.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/* Interfaces for converting to/from internal C++ formats. */
+@interface RTC_OBJC_TYPE (RTCCodecSpecificInfoH265) ()
+
+- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.h

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+/* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.h. */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCCodecSpecificInfo.h"
+#import "RTCMacros.h"
+
+/** Class for H265 specific config. */
+typedef NS_ENUM(NSUInteger, RTCH265PacketizationMode) {
+  RTCH265PacketizationModeNonInterleaved = 0,  // Mode 1 - STAP-A, FU-A is allowed
+  RTCH265PacketizationModeSingleNalUnit        // Mode 0 - only single NALU allowed
+};
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCCodecSpecificInfoH265) : NSObject <RTC_OBJC_TYPE(RTCCodecSpecificInfo)>
+
+@property(nonatomic, assign) RTCH265PacketizationMode packetizationMode;
+
+@end

sdk/objc/components/video_codec/RTCCodecSpecificInfoH265.mm

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+ /* This file is borrowed from sdk/objc/components/video_codec/RTCCodecSpecificInfoH264.mm */
+
+#import "RTCCodecSpecificInfoH265+Private.h"
+
+// H265 specific settings.
+@implementation RTC_OBJC_TYPE (RTCCodecSpecificInfoH265)
+
+@synthesize packetizationMode = _packetizationMode;
+
+- (webrtc::CodecSpecificInfo)nativeCodecSpecificInfo {
+  webrtc::CodecSpecificInfo codecSpecificInfo;
+  codecSpecificInfo.codecType = webrtc::kVideoCodecH265;
+  codecSpecificInfo.codecSpecific.H264.packetization_mode =
+      (webrtc::H264PacketizationMode)_packetizationMode;
+
+  return codecSpecificInfo;
+}
+
+@end

sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m

@@ -15,6 +15,8 @@
 #import "api/video_codec/RTCVideoCodecConstants.h"
 #import "api/video_codec/RTCVideoDecoderVP8.h"
 #import "api/video_codec/RTCVideoDecoderVP9.h"
+#import "RTCH265ProfileLevelId.h"
+#import "RTCVideoDecoderH265.h"
 #import "base/RTCVideoCodecInfo.h"
 
 #if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY)
@@ -42,13 +44,17 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory)
       [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:RTC_CONSTANT_TYPE(RTCVideoCodecH264Name)
                                                   parameters:constrainedBaselineParams];
 
+  RTC_OBJC_TYPE(RTCVideoCodecInfo) *h265Info =
+      [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:RTC_CONSTANT_TYPE(RTCVideoCodecH265Name)];
+
   RTC_OBJC_TYPE(RTCVideoCodecInfo) *vp8Info =
       [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:RTC_CONSTANT_TYPE(RTCVideoCodecVp8Name)];
 
   NSMutableArray<RTC_OBJC_TYPE(RTCVideoCodecInfo) *> *result = [@[
     constrainedHighInfo,
     constrainedBaselineInfo,
     vp8Info,
+    h265Info,
   ] mutableCopy];
 
   if ([RTC_OBJC_TYPE(RTCVideoDecoderVP9) isSupported]) {
@@ -68,6 +74,8 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory)
     return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] init];
   } else if ([info.name isEqualToString:RTC_CONSTANT_TYPE(RTCVideoCodecVp8Name)]) {
     return [RTC_OBJC_TYPE(RTCVideoDecoderVP8) vp8Decoder];
+  } else if ([info.name isEqualToString:RTC_CONSTANT_TYPE(RTCVideoCodecH265Name)]) {
+    return [[RTC_OBJC_TYPE(RTCVideoDecoderH265) alloc] init];
   } else if ([info.name isEqualToString:RTC_CONSTANT_TYPE(RTCVideoCodecVp9Name)] &&
              [RTC_OBJC_TYPE(RTCVideoDecoderVP9) isSupported]) {
     return [RTC_OBJC_TYPE(RTCVideoDecoderVP9) vp9Decoder];

sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m

@@ -15,6 +15,8 @@
 #import "api/video_codec/RTCVideoCodecConstants.h"
 #import "api/video_codec/RTCVideoEncoderVP8.h"
 #import "api/video_codec/RTCVideoEncoderVP9.h"
+#import "RTCH265ProfileLevelId.h"
+#import "RTCVideoEncoderH265.h"
 #import "base/RTCVideoCodecInfo.h"
 
 #if defined(RTC_USE_LIBAOM_AV1_ENCODER)
@@ -75,6 +77,10 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory)
   } else if ([info.name isEqualToString:RTC_CONSTANT_TYPE(RTCVideoCodecVp9Name)] &&
              [RTC_OBJC_TYPE(RTCVideoEncoderVP9) isSupported]) {
     return [RTC_OBJC_TYPE(RTCVideoEncoderVP9) vp9Encoder];
+  } else if (@available(iOS 11, *)) {
+    if ([info.name isEqualToString:RTC_CONSTANT_TYPE(RTCVideoCodecH265Name)]) {
+      return [[RTC_OBJC_TYPE(RTCVideoEncoderH265) alloc] initWithCodecInfo:info];
+    }
   }
 
 #if defined(RTC_USE_LIBAOM_AV1_ENCODER)

sdk/objc/components/video_codec/RTCH265ProfileLevelId.h

@@ -0,0 +1,16 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+
+RTC_OBJC_EXPORT extern NSString *const RTC_CONSTANT_TYPE(RTCVideoCodecH265Name);
+RTC_OBJC_EXPORT extern NSString *const RTC_CONSTANT_TYPE(RTCLevel31Main);

sdk/objc/components/video_codec/RTCH265ProfileLevelId.mm

@@ -0,0 +1,18 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#import "RTCH265ProfileLevelId.h"
+
+#include "media/base/media_constants.h"
+
+NSString *const RTC_CONSTANT_TYPE(RTCVideoCodecH265Name) = @"H265";
+// TODO(jianjunz): This is value is not correct.
+NSString *const RTC_CONSTANT_TYPE(RTCLevel31Main) = @"4d001f";

sdk/objc/components/video_codec/RTCVideoDecoderH264.mm

@@ -255,11 +255,7 @@ - (int)resetDecompressionSession {
 
 - (void)configureDecompressionSession {
   RTC_DCHECK(_decompressionSession);
-#if defined(WEBRTC_IOS)
-  VTSessionSetProperty(_decompressionSession,
-                       kVTDecompressionPropertyKey_RealTime,
-                       kCFBooleanTrue);
-#endif
+  VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
 }
 
 - (void)destroyDecompressionSession {

sdk/objc/components/video_codec/RTCVideoDecoderH265.h

@@ -0,0 +1,23 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+#import "RTCVideoDecoder.h"
+
+RTC_OBJC_EXPORT
+@interface RTC_OBJC_TYPE (RTCVideoDecoderH265) : NSObject <RTC_OBJC_TYPE(RTCVideoDecoder)>
+- (NSInteger)setHVCCFormat:(const uint8_t *)data size:(size_t)size width:(uint16_t)width height:(uint16_t)height;
+- (NSInteger)decodeData:(const uint8_t *)data
+    size:(size_t)size
+    timeStamp:(int64_t)timeStamp;
+- (void)flush;
+@end