Fix screen-share memory leak (#174)

* fix: memory leak in VideoTrackDesktopSource

* fix: memory leak in VideoTrackDesktopSource

* fix: compile on Windows with clang-cl

Author: devopvoid

webrtc-jni/pom.xml

@@ -146,6 +146,12 @@
 		</profile>
 		<profile>
 			<id>windows-clang</id>
+			<activation>
+				<os>
+					<family>windows</family>
+					<arch>amd64</arch>
+				</os>
+			</activation>
 			<properties>
 				<cmake.clang>-T ClangCL</cmake.clang>
 			</properties>

webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h

@@ -18,13 +18,17 @@
 #define JNI_WEBRTC_MEDIA_VIDEO_TRACK_DESKTOP_SOURCE_H_
 
 #include "api/video/i420_buffer.h"
-#include "media/base/adapted_video_track_source.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_sink_interface.h"
+#include "media/base/video_adapter.h"
+#include "media/base/video_broadcaster.h"
+#include "pc/video_track_source.h"
 #include "modules/desktop_capture/desktop_capturer.h"
 #include "rtc_base/thread.h"
 
 namespace jni
 {
-	class VideoTrackDesktopSource : public rtc::AdaptedVideoTrackSource, public webrtc::DesktopCapturer::Callback
+	class VideoTrackDesktopSource : public webrtc::VideoTrackSource, public webrtc::DesktopCapturer::Callback
 	{
 		public:
 			VideoTrackDesktopSource();
@@ -39,7 +43,16 @@ namespace jni
 			void stop();
 			void terminate();
 
-			// AdaptedVideoTrackSource implementation.
+			// VideoSourceInterface implementation.
+			void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame> * sink, const rtc::VideoSinkWants & wants) override;
+			void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame> * sink) override;
+
+			// VideoTrackSource implementation.
+			rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override;
+
+			// VideoTrackSourceInterface implementation.
+			bool GetStats(webrtc::VideoTrackSourceInterface::Stats * stats) override;
+			void ProcessConstraints(const webrtc::VideoTrackSourceConstraints & constraints) override;
 			virtual bool is_screencast() const override;
 			virtual std::optional<bool> needs_denoising() const override;
 			SourceState state() const override;
@@ -48,9 +61,14 @@ namespace jni
 			// DesktopCapturer::Callback implementation.
 			void OnCaptureResult(webrtc::DesktopCapturer::Result result, std::unique_ptr<webrtc::DesktopFrame> frame) override;
 
+		protected:
+			bool AdaptFrame(int width, int height, int64_t time_us, int* out_width, int* out_height, int* crop_width, int* crop_height, int* crop_x, int* crop_y);
+			void OnFrameDropped();
+
 		private:
 			void capture();
 			void process(std::unique_ptr<webrtc::DesktopFrame> & frame);
+			void updateVideoAdapter();
 
 		private:
 			uint16_t frameRate;
@@ -59,6 +77,9 @@ namespace jni
 
 			webrtc::DesktopSize maxFrameSize;
 
+			rtc::VideoBroadcaster broadcaster;
+			cricket::VideoAdapter videoAdapter;
+
 			webrtc::MediaSourceInterface::SourceState sourceState;
 
 			webrtc::DesktopCapturer::SourceId sourceId;
@@ -69,6 +90,9 @@ namespace jni
 			std::unique_ptr<rtc::Thread> captureThread;
 
 			rtc::scoped_refptr<webrtc::I420Buffer> buffer;
+
+			webrtc::Mutex statsMutex;
+			std::optional<Stats> stats RTC_GUARDED_BY(statsMutex);
 	};
 }
 

webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp

@@ -33,7 +33,7 @@
 namespace jni
 {
 	VideoTrackDesktopSource::VideoTrackDesktopSource() :
-		AdaptedVideoTrackSource(),
+		VideoTrackSource(/*remote=*/false),
 		frameRate(20),
 		isCapturing(false),
 		focusSelectedSource(true),
@@ -88,29 +88,104 @@ namespace jni
 		}
 	}
 
+	void VideoTrackDesktopSource::AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, const rtc::VideoSinkWants& wants)
+	{
+		if (wants.is_active) {
+			broadcaster.AddOrUpdateSink(sink, wants);
+
+			updateVideoAdapter();
+		}
+	}
+
+	void VideoTrackDesktopSource::RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink)
+	{
+		broadcaster.RemoveSink(sink);
+
+		updateVideoAdapter();
+	}
+
+	void VideoTrackDesktopSource::updateVideoAdapter()
+	{
+		videoAdapter.OnSinkWants(broadcaster.wants());
+	}
+
+	void VideoTrackDesktopSource::OnFrameDropped()
+	{
+		broadcaster.OnDiscardedFrame();
+	}
+
+	rtc::VideoSourceInterface<webrtc::VideoFrame>* VideoTrackDesktopSource::source()
+	{
+		return this;
+	}
+
 	void VideoTrackDesktopSource::terminate()
 	{
 		// Notify the track that we are permanently done.
 		sourceState = kEnded;
 		FireOnChanged();
 	}
 
-	bool VideoTrackDesktopSource::is_screencast() const {
+	bool VideoTrackDesktopSource::GetStats(webrtc::VideoTrackSourceInterface::Stats * stats)
+	{
+		webrtc::MutexLock lock(&statsMutex);
+
+		if (!stats) {
+			return false;
+		}
+
+		*stats = *stats;
+
+		return true;
+	}
+
+	void VideoTrackDesktopSource::ProcessConstraints(const webrtc::VideoTrackSourceConstraints & constraints)
+	{
+		broadcaster.ProcessConstraints(constraints);
+	}
+
+	bool VideoTrackDesktopSource::is_screencast() const
+	{
 		return true;
 	}
 
-	std::optional<bool> VideoTrackDesktopSource::needs_denoising() const {
+	std::optional<bool> VideoTrackDesktopSource::needs_denoising() const
+	{
 		return false;
 	}
 
-	webrtc::MediaSourceInterface::SourceState VideoTrackDesktopSource::state() const {
+	webrtc::MediaSourceInterface::SourceState VideoTrackDesktopSource::state() const
+	{
 		return sourceState;
 	}
 
-	bool VideoTrackDesktopSource::remote() const {
+	bool VideoTrackDesktopSource::remote() const
+	{
 		return false;
 	}
 
+	bool VideoTrackDesktopSource::AdaptFrame(int width, int height, int64_t time_us, int* out_width, int* out_height, int* crop_width, int* crop_height, int* crop_x, int* crop_y)
+	{
+		{
+			webrtc::MutexLock lock(&statsMutex);
+			stats = Stats{ width, height };
+		}
+
+		if (!broadcaster.frame_wanted()) {
+			return false;
+		}
+
+		if (!videoAdapter.AdaptFrameResolution(width, height, time_us * rtc::kNumNanosecsPerMicrosec, crop_width, crop_height, out_width, out_height)) {
+			broadcaster.OnDiscardedFrame();
+			return false;
+		}
+
+		*crop_x = (width - *crop_width) / 2;
+		*crop_y = (height - *crop_height) / 2;
+		
+		return true;
+	}
+
 	void VideoTrackDesktopSource::OnCaptureResult(webrtc::DesktopCapturer::Result result, std::unique_ptr<webrtc::DesktopFrame> frame)
 	{
 		if (result != webrtc::DesktopCapturer::Result::SUCCESS) {
@@ -149,8 +224,8 @@ namespace jni
 		int width = frame->size().width();
 		int height = frame->size().height();
 
-		int adapted_width;
-		int adapted_height;
+		int adapted_width = width;
+		int adapted_height = height;
 
 		int crop_x = 0;
 		int crop_y = 0;
@@ -216,18 +291,19 @@ namespace jni
 
 				scaled_buffer->ScaleFrom(*buffer);
 
-				OnFrame(webrtc::VideoFrame::Builder()
+				broadcaster.OnFrame(webrtc::VideoFrame::Builder()
 					.set_video_frame_buffer(scaled_buffer)
 					.set_rotation(webrtc::kVideoRotation_0)
 					.set_timestamp_us(time)
 					.build());
 			}
 			else {
 				// No adaptations needed, just return the frame as is.
-				OnFrame(webrtc::VideoFrame::Builder()
+				broadcaster.OnFrame(webrtc::VideoFrame::Builder()
 					.set_video_frame_buffer(buffer)
 					.set_rotation(webrtc::kVideoRotation_0)
 					.set_timestamp_us(time)
+					.set_timestamp_rtp(0)
 					.build());
 			}
 		}