feat: let a video processor change camera and desktop frames

docs/guide/video/camera-capture.md

@@ -295,6 +295,43 @@ This scaling process works as follows:
 
 Note that scaling is a computationally expensive operation, especially for high-resolution frames. Consider the performance implications when choosing target dimensions and how frequently you scale frames.
 
+### Changing Frames Before They Are Sent
+
+A `VideoProcessor` sits between a camera or desktop source and its tracks, so it can change what is sent and shown: blur a background, draw an overlay, mirror the picture. The source passes each frame to `onFrameCaptured()` after adapting it to what its tracks ask for, so frames already have the size and rate that will be sent. The processor delivers the frames to send to the sink it gets in `setSink()`: the same frame, a new one, or none to drop it.
+
+```java
+videoSource.setVideoProcessor(new VideoProcessor() {
+    private VideoTrackSink sink;
+
+    @Override
+    public void setSink(VideoTrackSink sink) {
+        this.sink = sink;
+    }
+
+    @Override
+    public void onFrameCaptured(VideoFrame frame) {
+        // Captured frames are I420; toI420() returns the same buffer.
+        I420Buffer input = frame.buffer.toI420();
+        NativeI420Buffer output = NativeI420Buffer.allocate(input.getWidth(), input.getHeight());
+
+        // ... write the changed picture into output's planes ...
+
+        VideoFrame processed = new VideoFrame(output, frame.rotation, frame.timestampNs);
+        sink.onVideoFrame(processed);
+        processed.release();
+    }
+});
+```
+
+Rules that keep this correct:
+
+- The frame passed to `onFrameCaptured()` is released when the method returns. To deliver it later, or from another thread, call `frame.retain()` first and `frame.release()` when done.
+- The sink does not take over a delivered frame's reference, so release a frame you created or retained after delivering it.
+- The pixels of a captured frame may be shared with the source. Write changes into a new buffer, as above, instead of into the captured one.
+- `onFrameCaptured()` runs on the capture thread and holds up capturing while it runs. An exception it throws goes to that thread's uncaught exception handler, and the frame is dropped.
+
+`setVideoProcessor(null)` removes the processor. A `CustomVideoSource` does not take a processor: change the frames before pushing them.
+
 ## Best Practices
 
 When implementing camera capture in your application, consider these best practices:

docs/guide/video/desktop-capture.md

@@ -147,6 +147,10 @@ videoSource.stop();
 videoSource.dispose();
 ```
 
+### Processing Frames
+
+A `VideoDesktopSource` can pass its frames through a `VideoProcessor` before they reach its tracks, for example to blank out a part of the screen. It works as for a camera; see [Changing Frames Before They Are Sent](/guide/video/camera-capture#changing-frames-before-they-are-sent).
+
 ### Handling Source Changes
 
 If desktop sources might change during your application's lifecycle (e.g., new windows opening or screens connecting), you should periodically refresh the source list:

webrtc-jni/src/main/cpp/include/media/video/VideoProcessorHost.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JNI_WEBRTC_MEDIA_VIDEO_VIDEO_PROCESSOR_HOST_H_
+#define JNI_WEBRTC_MEDIA_VIDEO_VIDEO_PROCESSOR_HOST_H_
+
+#include "JavaClass.h"
+#include "JavaRef.h"
+
+#include "api/video/video_frame.h"
+
+#include <jni.h>
+#include <memory>
+#include <mutex>
+
+namespace jni
+{
+	// A video source that can run its frames through a Java VideoProcessor.
+	// The source calls ProcessFrame with each frame it would otherwise
+	// deliver; the processor hands the frames to send back through the Java
+	// sink, which ends up in DeliverProcessedFrame.
+	class VideoProcessorHost
+	{
+		public:
+			virtual ~VideoProcessorHost() = default;
+
+			// Sets the Java processor, or removes it when null. A frame that is
+			// being processed meanwhile still reaches the previous processor.
+			void SetVideoProcessor(JNIEnv * env, jobject processor);
+
+			// Hands the frame to the processor. Returns false when there is
+			// none, so that the source delivers the frame itself. Never throws.
+			bool ProcessFrame(const webrtc::VideoFrame & frame);
+
+			// Delivers a frame the processor produced to the source's sinks.
+			virtual void DeliverProcessedFrame(const webrtc::VideoFrame & frame) = 0;
+
+		private:
+			class JavaVideoProcessorClass : public JavaClass
+			{
+				public:
+					explicit JavaVideoProcessorClass(JNIEnv * env);
+
+					jmethodID onFrameCaptured;
+			};
+
+			std::mutex mutex;
+			std::shared_ptr<JavaGlobalRef<jobject>> processor;
+	};
+}
+
+#endif

webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h

@@ -22,9 +22,11 @@
 #include "modules/desktop_capture/desktop_capturer.h"
 #include "rtc_base/platform_thread.h"
 
+#include "media/video/VideoProcessorHost.h"
+
 namespace jni
 {
-	class VideoTrackDesktopSource : public webrtc::AdaptedVideoTrackSource, public webrtc::DesktopCapturer::Callback
+	class VideoTrackDesktopSource : public webrtc::AdaptedVideoTrackSource, public webrtc::DesktopCapturer::Callback, public VideoProcessorHost
 	{
         public:
             VideoTrackDesktopSource();
@@ -48,6 +50,9 @@ namespace jni
             // DesktopCapturer::Callback implementation.
             void OnCaptureResult(webrtc::DesktopCapturer::Result result, std::unique_ptr<webrtc::DesktopFrame> frame) override;
 
+            // VideoProcessorHost implementation.
+            void DeliverProcessedFrame(const webrtc::VideoFrame & frame) override;
+
         private:
             void capture();
             void process(std::unique_ptr<webrtc::DesktopFrame>& frame);

webrtc-jni/src/main/cpp/include/media/video/VideoTrackDeviceSource.h

@@ -26,12 +26,13 @@
 #include "modules/video_capture/video_capture.h"
 #include "modules/video_capture/video_capture_defines.h"
 
+#include "media/video/VideoProcessorHost.h"
 #include "media/video/VideoTrackDeviceSourceBase.h"
 #include "media/video/VideoDevice.h"
 
 namespace jni
 {
-	class VideoTrackDeviceSource : public webrtc::VideoTrackSource, public webrtc::VideoSinkInterface<webrtc::VideoFrame>, public VideoTrackDeviceSourceBase
+	class VideoTrackDeviceSource : public webrtc::VideoTrackSource, public webrtc::VideoSinkInterface<webrtc::VideoFrame>, public VideoTrackDeviceSourceBase, public VideoProcessorHost
 	{
 		public:
 			VideoTrackDeviceSource();
@@ -51,6 +52,9 @@ namespace jni
 			// VideoSinkInterface implementation.
 			void OnFrame(const webrtc::VideoFrame & frame) override;
 
+			// VideoProcessorHost implementation.
+			void DeliverProcessedFrame(const webrtc::VideoFrame & frame) override;
+
 		private:
 			bool startCapture();
 			void updateVideoAdapter();

webrtc-jni/src/main/cpp/include/media/video/macos/VideoTrackDeviceSourceMac.h

@@ -25,6 +25,7 @@
 #include "sdk/objc/base/RTCMacros.h"
 #include "sdk/objc/components/capturer/RTCCameraVideoCapturer.h"
 
+#include "media/video/VideoProcessorHost.h"
 #include "media/video/VideoTrackDeviceSourceBase.h"
 
 
@@ -35,7 +36,7 @@
 
 namespace jni
 {
-	class VideoTrackDeviceSourceMac : public webrtc::AdaptedVideoTrackSource, public VideoTrackDeviceSourceBase
+	class VideoTrackDeviceSourceMac : public webrtc::AdaptedVideoTrackSource, public VideoTrackDeviceSourceBase, public VideoProcessorHost
 	{
 		public:
 			VideoTrackDeviceSourceMac();
@@ -53,6 +54,9 @@ namespace jni
 
             void OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame);
 
+            // VideoProcessorHost implementation.
+            void DeliverProcessedFrame(const webrtc::VideoFrame & frame) override;
+
 		private:
 			void destroy();
 

webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp

@@ -88,12 +88,17 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_sto
 	}
 }
 
-JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_dispose
+JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_disposeInternal
 (JNIEnv * env, jobject caller)
 {
 	jni::VideoTrackDesktopSource * videoSource = GetHandle<jni::VideoTrackDesktopSource>(env, caller);
 	CHECK_HANDLE(videoSource);
 
+	// A track may keep the source alive; it must not reach the processor
+	// any more.
+	videoSource->SetVideoProcessor(env, nullptr);
+	SetHandle<std::nullptr_t>(env, caller, "processorHostHandle", nullptr);
+
 	webrtc::RefCountReleaseStatus status = videoSource->Release();
 
 	if (status != webrtc::RefCountReleaseStatus::kDroppedLastRef) {
@@ -110,5 +115,6 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_ini
 {
 	webrtc::scoped_refptr<jni::VideoTrackDesktopSource> videoSource = webrtc::make_ref_counted<jni::VideoTrackDesktopSource>();
 
+	SetHandle(env, caller, "processorHostHandle", static_cast<jni::VideoProcessorHost *>(videoSource.get()));
 	SetHandle(env, caller, videoSource.release());
 }

webrtc-jni/src/main/cpp/src/JNI_VideoDeviceSource.cpp

@@ -116,7 +116,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_stop
 	}
 }
 
-JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_dispose
+JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_disposeInternal
 (JNIEnv * env, jobject caller)
 {
 #ifdef __APPLE__
@@ -127,6 +127,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_disp
 
 	CHECK_HANDLE(videoSource);
 
+	// A track may keep the source alive; it must not reach the processor
+	// any more.
+	videoSource->SetVideoProcessor(env, nullptr);
+	SetHandle<std::nullptr_t>(env, caller, "processorHostHandle", nullptr);
+
 	webrtc::RefCountReleaseStatus status = videoSource->Release();
 
 	if (status != webrtc::RefCountReleaseStatus::kDroppedLastRef) {
@@ -143,9 +148,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_init
 {
 #ifdef __APPLE__
     webrtc::scoped_refptr<jni::VideoTrackDeviceSourceMac> videoSource_ = webrtc::make_ref_counted<jni::VideoTrackDeviceSourceMac>();
+    SetHandle(env, caller, "processorHostHandle", static_cast<jni::VideoProcessorHost *>(videoSource_.get()));
     SetHandle(env, caller, videoSource_.release());
 #else
     webrtc::scoped_refptr<jni::VideoTrackDeviceSource> videoSource = webrtc::make_ref_counted<jni::VideoTrackDeviceSource>();
+    SetHandle(env, caller, "processorHostHandle", static_cast<jni::VideoProcessorHost *>(videoSource.get()));
     SetHandle(env, caller, videoSource.release());
 #endif
 }