diff --git a/docs/guide/video/camera-capture.md b/docs/guide/video/camera-capture.md index 2fffe0697..6b8f44694 100644 --- a/docs/guide/video/camera-capture.md +++ b/docs/guide/video/camera-capture.md @@ -295,6 +295,43 @@ This scaling process works as follows: Note that scaling is a computationally expensive operation, especially for high-resolution frames. Consider the performance implications when choosing target dimensions and how frequently you scale frames. +### Changing Frames Before They Are Sent + +A `VideoProcessor` sits between a camera or desktop source and its tracks, so it can change what is sent and shown: blur a background, draw an overlay, mirror the picture. The source passes each frame to `onFrameCaptured()` after adapting it to what its tracks ask for, so frames already have the size and rate that will be sent. The processor delivers the frames to send to the sink it gets in `setSink()`: the same frame, a new one, or none to drop it. + +```java +videoSource.setVideoProcessor(new VideoProcessor() { + private VideoTrackSink sink; + + @Override + public void setSink(VideoTrackSink sink) { + this.sink = sink; + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + // Captured frames are I420; toI420() returns the same buffer. + I420Buffer input = frame.buffer.toI420(); + NativeI420Buffer output = NativeI420Buffer.allocate(input.getWidth(), input.getHeight()); + + // ... write the changed picture into output's planes ... + + VideoFrame processed = new VideoFrame(output, frame.rotation, frame.timestampNs); + sink.onVideoFrame(processed); + processed.release(); + } +}); +``` + +Rules that keep this correct: + +- The frame passed to `onFrameCaptured()` is released when the method returns. To deliver it later, or from another thread, call `frame.retain()` first and `frame.release()` when done. +- The sink does not take over a delivered frame's reference, so release a frame you created or retained after delivering it. +- The pixels of a captured frame may be shared with the source. Write changes into a new buffer, as above, instead of into the captured one. +- `onFrameCaptured()` runs on the capture thread and holds up capturing while it runs. An exception it throws goes to that thread's uncaught exception handler, and the frame is dropped. + +`setVideoProcessor(null)` removes the processor. A `CustomVideoSource` does not take a processor: change the frames before pushing them. + ## Best Practices When implementing camera capture in your application, consider these best practices: diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index a0ebc5e9a..067060b3f 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -147,6 +147,10 @@ videoSource.stop(); videoSource.dispose(); ``` +### Processing Frames + +A `VideoDesktopSource` can pass its frames through a `VideoProcessor` before they reach its tracks, for example to blank out a part of the screen. It works as for a camera; see [Changing Frames Before They Are Sent](/guide/video/camera-capture#changing-frames-before-they-are-sent). + ### Handling Source Changes If desktop sources might change during your application's lifecycle (e.g., new windows opening or screens connecting), you should periodically refresh the source list: diff --git a/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h b/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h index b255b3ec0..ed26a79aa 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h +++ b/webrtc-jni/src/main/cpp/include/JNI_VideoDesktopSource.h @@ -57,10 +57,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_video_VideoDesktopSource - * Method: dispose + * Method: disposeInternal * Signature: ()V */ - JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_dispose + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_disposeInternal (JNIEnv*, jobject); /* diff --git a/webrtc-jni/src/main/cpp/include/JNI_VideoDeviceSource.h b/webrtc-jni/src/main/cpp/include/JNI_VideoDeviceSource.h index 3c1b3007c..b0de11cce 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_VideoDeviceSource.h +++ b/webrtc-jni/src/main/cpp/include/JNI_VideoDeviceSource.h @@ -41,10 +41,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_video_VideoDeviceSource - * Method: dispose + * Method: disposeInternal * Signature: ()V */ - JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_dispose + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_disposeInternal (JNIEnv *, jobject); /* diff --git a/webrtc-jni/src/main/cpp/include/JNI_VideoTrackSource.h b/webrtc-jni/src/main/cpp/include/JNI_VideoTrackSource.h new file mode 100644 index 000000000..2468f8ba6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_VideoTrackSource.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class dev_onvoid_webrtc_media_video_VideoTrackSource */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_VideoTrackSource +#define _Included_dev_onvoid_webrtc_media_video_VideoTrackSource +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_VideoTrackSource + * Method: setVideoProcessorNative + * Signature: (Ldev/onvoid/webrtc/media/video/VideoProcessor;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrackSource_setVideoProcessorNative + (JNIEnv *, jobject, jobject); + + /* + * Class: dev_onvoid_webrtc_media_video_VideoTrackSource + * Method: deliverProcessedFrame + * Signature: (Ldev/onvoid/webrtc/media/video/VideoFrame;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrackSource_deliverProcessedFrame + (JNIEnv *, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/VideoProcessorHost.h b/webrtc-jni/src/main/cpp/include/media/video/VideoProcessorHost.h new file mode 100644 index 000000000..24e7d04b3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/VideoProcessorHost.h @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_VIDEO_PROCESSOR_HOST_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_VIDEO_PROCESSOR_HOST_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/video_frame.h" + +#include +#include +#include + +namespace jni +{ + // A video source that can run its frames through a Java VideoProcessor. + // The source calls ProcessFrame with each frame it would otherwise + // deliver; the processor hands the frames to send back through the Java + // sink, which ends up in DeliverProcessedFrame. + class VideoProcessorHost + { + public: + virtual ~VideoProcessorHost() = default; + + // Sets the Java processor, or removes it when null. A frame that is + // being processed meanwhile still reaches the previous processor. + void SetVideoProcessor(JNIEnv * env, jobject processor); + + // Hands the frame to the processor. Returns false when there is + // none, so that the source delivers the frame itself. Never throws. + bool ProcessFrame(const webrtc::VideoFrame & frame); + + // Delivers a frame the processor produced to the source's sinks. + virtual void DeliverProcessedFrame(const webrtc::VideoFrame & frame) = 0; + + private: + class JavaVideoProcessorClass : public JavaClass + { + public: + explicit JavaVideoProcessorClass(JNIEnv * env); + + jmethodID onFrameCaptured; + }; + + std::mutex mutex; + std::shared_ptr> processor; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h b/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h index 04d154e48..282d5adff 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h +++ b/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDesktopSource.h @@ -22,9 +22,11 @@ #include "modules/desktop_capture/desktop_capturer.h" #include "rtc_base/platform_thread.h" +#include "media/video/VideoProcessorHost.h" + namespace jni { - class VideoTrackDesktopSource : public webrtc::AdaptedVideoTrackSource, public webrtc::DesktopCapturer::Callback + class VideoTrackDesktopSource : public webrtc::AdaptedVideoTrackSource, public webrtc::DesktopCapturer::Callback, public VideoProcessorHost { public: VideoTrackDesktopSource(); @@ -48,6 +50,9 @@ namespace jni // DesktopCapturer::Callback implementation. void OnCaptureResult(webrtc::DesktopCapturer::Result result, std::unique_ptr frame) override; + // VideoProcessorHost implementation. + void DeliverProcessedFrame(const webrtc::VideoFrame & frame) override; + private: void capture(); void process(std::unique_ptr& frame); diff --git a/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDeviceSource.h b/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDeviceSource.h index 067f1e143..e06bb0877 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDeviceSource.h +++ b/webrtc-jni/src/main/cpp/include/media/video/VideoTrackDeviceSource.h @@ -26,12 +26,13 @@ #include "modules/video_capture/video_capture.h" #include "modules/video_capture/video_capture_defines.h" +#include "media/video/VideoProcessorHost.h" #include "media/video/VideoTrackDeviceSourceBase.h" #include "media/video/VideoDevice.h" namespace jni { - class VideoTrackDeviceSource : public webrtc::VideoTrackSource, public webrtc::VideoSinkInterface, public VideoTrackDeviceSourceBase + class VideoTrackDeviceSource : public webrtc::VideoTrackSource, public webrtc::VideoSinkInterface, public VideoTrackDeviceSourceBase, public VideoProcessorHost { public: VideoTrackDeviceSource(); @@ -51,6 +52,9 @@ namespace jni // VideoSinkInterface implementation. void OnFrame(const webrtc::VideoFrame & frame) override; + // VideoProcessorHost implementation. + void DeliverProcessedFrame(const webrtc::VideoFrame & frame) override; + private: bool startCapture(); void updateVideoAdapter(); diff --git a/webrtc-jni/src/main/cpp/include/media/video/macos/VideoTrackDeviceSourceMac.h b/webrtc-jni/src/main/cpp/include/media/video/macos/VideoTrackDeviceSourceMac.h index 5ec9a32f7..ed3b5162f 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/macos/VideoTrackDeviceSourceMac.h +++ b/webrtc-jni/src/main/cpp/include/media/video/macos/VideoTrackDeviceSourceMac.h @@ -25,6 +25,7 @@ #include "sdk/objc/base/RTCMacros.h" #include "sdk/objc/components/capturer/RTCCameraVideoCapturer.h" +#include "media/video/VideoProcessorHost.h" #include "media/video/VideoTrackDeviceSourceBase.h" @@ -35,7 +36,7 @@ namespace jni { - class VideoTrackDeviceSourceMac : public webrtc::AdaptedVideoTrackSource, public VideoTrackDeviceSourceBase + class VideoTrackDeviceSourceMac : public webrtc::AdaptedVideoTrackSource, public VideoTrackDeviceSourceBase, public VideoProcessorHost { public: VideoTrackDeviceSourceMac(); @@ -53,6 +54,9 @@ namespace jni void OnCapturedFrame(RTC_OBJC_TYPE(RTCVideoFrame) * frame); + // VideoProcessorHost implementation. + void DeliverProcessedFrame(const webrtc::VideoFrame & frame) override; + private: void destroy(); diff --git a/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp index 2dfb04d47..1cb2e1fe4 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_VideoDesktopSource.cpp @@ -88,12 +88,17 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_sto } } -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_dispose +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_disposeInternal (JNIEnv * env, jobject caller) { jni::VideoTrackDesktopSource * videoSource = GetHandle(env, caller); CHECK_HANDLE(videoSource); + // A track may keep the source alive; it must not reach the processor + // any more. + videoSource->SetVideoProcessor(env, nullptr); + SetHandle(env, caller, "processorHostHandle", nullptr); + webrtc::RefCountReleaseStatus status = videoSource->Release(); if (status != webrtc::RefCountReleaseStatus::kDroppedLastRef) { @@ -110,5 +115,6 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDesktopSource_ini { webrtc::scoped_refptr videoSource = webrtc::make_ref_counted(); + SetHandle(env, caller, "processorHostHandle", static_cast(videoSource.get())); SetHandle(env, caller, videoSource.release()); } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/JNI_VideoDeviceSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_VideoDeviceSource.cpp index 3e2906396..820328ba9 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_VideoDeviceSource.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_VideoDeviceSource.cpp @@ -116,7 +116,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_stop } } -JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_dispose +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_disposeInternal (JNIEnv * env, jobject caller) { #ifdef __APPLE__ @@ -127,6 +127,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_disp CHECK_HANDLE(videoSource); + // A track may keep the source alive; it must not reach the processor + // any more. + videoSource->SetVideoProcessor(env, nullptr); + SetHandle(env, caller, "processorHostHandle", nullptr); + webrtc::RefCountReleaseStatus status = videoSource->Release(); if (status != webrtc::RefCountReleaseStatus::kDroppedLastRef) { @@ -143,9 +148,11 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoDeviceSource_init { #ifdef __APPLE__ webrtc::scoped_refptr videoSource_ = webrtc::make_ref_counted(); + SetHandle(env, caller, "processorHostHandle", static_cast(videoSource_.get())); SetHandle(env, caller, videoSource_.release()); #else webrtc::scoped_refptr videoSource = webrtc::make_ref_counted(); + SetHandle(env, caller, "processorHostHandle", static_cast(videoSource.get())); SetHandle(env, caller, videoSource.release()); #endif } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/JNI_VideoTrackSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_VideoTrackSource.cpp new file mode 100644 index 000000000..bc71da0d4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_VideoTrackSource.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_VideoTrackSource.h" +#include "api/VideoFrame.h" +#include "media/video/VideoProcessorHost.h" +#include "JavaClasses.h" +#include "JavaNullPointerException.h" +#include "JavaObject.h" +#include "JavaRef.h" +#include "JavaRuntimeException.h" +#include "JavaUtils.h" + +#include "api/video/video_frame.h" +#include "api/video/video_rotation.h" + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrackSource_setVideoProcessorNative +(JNIEnv * env, jobject caller, jobject jProcessor) +{ + jni::VideoProcessorHost * host = GetHandle(env, caller, "processorHostHandle"); + + if (host == nullptr) { + env->Throw(jni::JavaRuntimeException(env, "The video source does not support a VideoProcessor")); + return; + } + + try { + host->SetVideoProcessor(env, jProcessor); + } + catch (...) { + ThrowCxxJavaException(env); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_VideoTrackSource_deliverProcessedFrame +(JNIEnv * env, jobject caller, jobject jFrame) +{ + if (jFrame == nullptr) { + env->Throw(jni::JavaNullPointerException(env, "VideoFrame must not be null")); + return; + } + + jni::VideoProcessorHost * host = GetHandle(env, caller, "processorHostHandle"); + + // The source has been disposed. + if (host == nullptr) { + return; + } + + try { + const auto frameClass = jni::JavaClasses::get(env); + + const jni::JavaLocalRef frameRef(env, jFrame); + jni::JavaObject frame(env, frameRef); + + const jint rotation = frame.getInt(frameClass->rotation); + const jlong timestampNs = frame.getLong(frameClass->timestampNs); + + webrtc::scoped_refptr buffer = + jni::VideoFrame::toNativeBuffer(env, frame.getObject(frameClass->buffer)); + + if (buffer == nullptr) { + env->Throw(jni::JavaRuntimeException(env, "The buffer of the VideoFrame cannot be read")); + return; + } + + // Java only allows multiples of 90. + const int degrees = ((rotation % 360) + 360) % 360; + + host->DeliverProcessedFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(static_cast(degrees)) + .set_timestamp_us(timestampNs / 1000) + .build()); + } + catch (...) { + ThrowCxxJavaException(env); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoProcessorHost.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoProcessorHost.cpp new file mode 100644 index 000000000..da901e698 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoProcessorHost.cpp @@ -0,0 +1,117 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/VideoProcessorHost.h" +#include "api/VideoFrame.h" +#include "api/WebRTCUtils.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/i420_buffer.h" + +#include + +namespace jni +{ + void VideoProcessorHost::SetVideoProcessor(JNIEnv * env, jobject processor) + { + std::shared_ptr> next; + + if (processor != nullptr) { + next = std::make_shared>(env, processor); + } + + std::shared_ptr> previous; + + { + std::lock_guard lock(mutex); + + previous = std::exchange(this->processor, std::move(next)); + } + + // The previous processor is released here, outside the lock, unless a + // frame is still being processed by it. + } + + bool VideoProcessorHost::ProcessFrame(const webrtc::VideoFrame & frame) + { + std::shared_ptr> current; + + { + std::lock_guard lock(mutex); + + current = processor; + } + + if (!current) { + return false; + } + + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return true; + } + + try { + const auto processorClass = JavaClasses::get(env); + const auto frameClass = JavaClasses::get(env); + + webrtc::scoped_refptr buffer = frame.video_frame_buffer()->ToI420(); + + if (buffer != nullptr) { + // The Java frame holds a reference of its own, which is dropped + // once the processor returns. A processor that keeps the frame + // retains it. + buffer->AddRef(); + + struct Reference + { + webrtc::I420BufferInterface * buffer; + + ~Reference() + { + buffer->Release(); + } + } reference{ buffer.get() }; + + JavaLocalRef jBuffer = I420Buffer::toJava(env, buffer); + JavaLocalRef jFrame(env, env->NewObject(frameClass->cls, frameClass->ctor, jBuffer.get(), + static_cast(frame.rotation()), static_cast(frame.timestamp_us() * 1000))); + + if (jFrame.get() != nullptr) { + env->CallVoidMethod(current->get(), processorClass->onFrameCaptured, jFrame.get()); + } + } + } + catch (...) { + ThrowCxxJavaException(env); + } + + // The capture thread belongs to WebRTC or the capture module. + ReportPendingException(env); + + return true; + } + + VideoProcessorHost::JavaVideoProcessorClass::JavaVideoProcessorClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_VIDEO"VideoProcessor"); + + onFrameCaptured = GetMethod(env, cls, "onFrameCaptured", "(L" PKG_VIDEO "VideoFrame;)V"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index 3d9cf7217..af0fc3567 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -229,23 +229,36 @@ namespace jni scaled_buffer->ScaleFrom(*buffer); - OnFrame(webrtc::VideoFrame::Builder() + webrtc::VideoFrame scaled = webrtc::VideoFrame::Builder() .set_video_frame_buffer(scaled_buffer) .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(time) - .build()); + .build(); + + if (!ProcessFrame(scaled)) { + OnFrame(scaled); + } } else { // No adaptations needed, just return the frame as is. - OnFrame(webrtc::VideoFrame::Builder() + webrtc::VideoFrame unscaled = webrtc::VideoFrame::Builder() .set_video_frame_buffer(buffer) .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(time) - .build()); + .build(); + + if (!ProcessFrame(unscaled)) { + OnFrame(unscaled); + } } } } + void VideoTrackDesktopSource::DeliverProcessedFrame(const webrtc::VideoFrame & frame) + { + OnFrame(frame); + } + void VideoTrackDesktopSource::capture() { auto options = webrtc::DesktopCaptureOptions::CreateDefault(); diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDeviceSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDeviceSource.cpp index 555023d3c..4ca66a3e6 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDeviceSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDeviceSource.cpp @@ -189,19 +189,28 @@ namespace jni scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); - broadcaster.OnFrame(webrtc::VideoFrame::Builder() + webrtc::VideoFrame scaled = webrtc::VideoFrame::Builder() .set_video_frame_buffer(scaled_buffer) .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(frame.timestamp_us()) .set_id(frame.id()) - .build()); + .build(); + + if (!ProcessFrame(scaled)) { + broadcaster.OnFrame(scaled); + } } - else { + else if (!ProcessFrame(frame)) { // No adaptations needed, just return the frame as is. broadcaster.OnFrame(frame); } } + void VideoTrackDeviceSource::DeliverProcessedFrame(const webrtc::VideoFrame & frame) + { + broadcaster.OnFrame(frame); + } + webrtc::VideoSourceInterface * VideoTrackDeviceSource::source() { return this; diff --git a/webrtc-jni/src/main/cpp/src/media/video/macos/VideoTrackDeviceSourceMac.cpp b/webrtc-jni/src/main/cpp/src/media/video/macos/VideoTrackDeviceSourceMac.cpp index a59ba1ecd..72b513ca1 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/macos/VideoTrackDeviceSourceMac.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/macos/VideoTrackDeviceSourceMac.cpp @@ -205,10 +205,19 @@ namespace jni buffer = i420_buffer; } - OnFrame(webrtc::VideoFrame::Builder() + webrtc::VideoFrame captured = webrtc::VideoFrame::Builder() .set_video_frame_buffer(buffer) .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(translated_timestamp_us) - .build()); + .build(); + + if (!ProcessFrame(captured)) { + OnFrame(captured); + } + } + + void VideoTrackDeviceSourceMac::DeliverProcessedFrame(const webrtc::VideoFrame & frame) + { + OnFrame(frame); } } \ No newline at end of file diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/ProcessedFrameSink.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/ProcessedFrameSink.java new file mode 100644 index 000000000..cde3fb7d8 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/ProcessedFrameSink.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video; + +/** + * The sink a {@link VideoProcessor} delivers processed frames to. It passes + * them on to its source until it is detached, when the processor is replaced + * or the source is disposed, and drops them afterwards. + */ +final class ProcessedFrameSink implements VideoTrackSink { + + private VideoTrackSource source; + + + ProcessedFrameSink(VideoTrackSource source) { + this.source = source; + } + + @Override + public void onVideoFrame(VideoFrame frame) { + if (frame == null) { + throw new NullPointerException("VideoFrame must not be null"); + } + + // Holding the lock while delivering keeps the source from being + // disposed during the native call. + synchronized (this) { + if (source != null) { + source.deliverProcessedFrame(frame); + } + } + } + + synchronized void detach() { + source = null; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java index 5db3bc066..528e5af50 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDesktopSource.java @@ -34,8 +34,13 @@ public VideoDesktopSource() { public native void stop(); - public native void dispose(); + public void dispose() { + detachVideoProcessor(); + disposeInternal(); + } private native void initialize(); + private native void disposeInternal(); + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDeviceSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDeviceSource.java index d350675e6..4ed8011dc 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDeviceSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoDeviceSource.java @@ -32,8 +32,13 @@ public VideoDeviceSource() { public native void stop(); - public native void dispose(); + public void dispose() { + detachVideoProcessor(); + disposeInternal(); + } private native void initialize(); + private native void disposeInternal(); + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoProcessor.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoProcessor.java new file mode 100644 index 000000000..b1a3b351c --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoProcessor.java @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video; + +/** + * Processes the frames of a camera or desktop source before they reach its + * tracks, for example to blur a background, draw an overlay or mirror the + * picture. Set one with {@link VideoTrackSource#setVideoProcessor}. + *

+ * The source hands each frame to {@link #onFrameCaptured}, after it has + * adapted the frame to what its tracks ask for, so the frame has the size and + * comes at the rate that will be sent. The processor delivers the frames to + * send to the sink it was given with {@link #setSink}: the same frame, a new + * one, or none to drop it. It may deliver from another thread, later. + *

+ * The frame passed to {@code onFrameCaptured} is released once the method + * returns. To deliver it or read it afterwards, {@link VideoFrame#retain() + * retain} it first and {@link VideoFrame#release() release} it when done. + * The sink does not take over the reference of a delivered frame, so the + * processor releases a frame it created or retained after delivering it. The + * pixels of a captured frame may be shared with the source: to change them, + * write into a new buffer, such as one from {@link NativeI420Buffer#allocate}, + * and deliver a new frame. + *

+ * {@code onFrameCaptured} runs on the source's capture thread and holds up + * capturing while it runs. An exception it throws is reported to that + * thread's uncaught exception handler, and the frame is dropped. + * + * @author Alex Andres + */ +public interface VideoProcessor { + + /** + * Gives the processor the sink to deliver processed frames to. Called by + * {@link VideoTrackSource#setVideoProcessor} before the first frame + * arrives. The sink does nothing once the processor is replaced or the + * source is disposed. + * + * @param sink The sink that passes frames on to the source's tracks. + */ + void setSink(VideoTrackSink sink); + + /** + * Called for each frame the source captures and does not drop. + * + * @param frame The captured frame, valid until this method returns. + */ + void onFrameCaptured(VideoFrame frame); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrackSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrackSource.java index 5931c678c..64af0185c 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrackSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/VideoTrackSource.java @@ -23,8 +23,75 @@ */ public class VideoTrackSource extends MediaSource { + /* + * The native VideoProcessorHost of a source that can run a processor, set + * by the native source and cleared when it is disposed. + */ + @SuppressWarnings("unused") + private long processorHostHandle; + + private ProcessedFrameSink processorSink; + + protected VideoTrackSource() { } + /** + * Sets the processor that the frames of this source pass through before + * they reach its tracks, replacing any processor set before. {@code null} + * removes the processor, so frames go to the tracks unchanged. + *

+ * Camera and desktop sources support a processor. A {@link + * CustomVideoSource} does not: process the frames before pushing them. + * + * @param processor The processor, or {@code null}. + * + * @throws UnsupportedOperationException If this source does not support a + * processor, or has been disposed. + */ + public synchronized void setVideoProcessor(VideoProcessor processor) { + if (processorHostHandle == 0) { + throw new UnsupportedOperationException( + getClass().getSimpleName() + " does not support a VideoProcessor"); + } + + ProcessedFrameSink sink = null; + + if (processor != null) { + sink = new ProcessedFrameSink(this); + + processor.setSink(sink); + } + + setVideoProcessorNative(processor); + + // Frames the previous processor delivers from now on are dropped. + if (processorSink != null) { + processorSink.detach(); + } + + processorSink = sink; + } + + /** + * Detaches the sink of the current processor, so that it drops frames + * delivered while and after this source is disposed. Called by the + * sources that support a processor before they dispose of their native + * source, which removes the processor. + */ + synchronized void detachVideoProcessor() { + if (processorSink != null) { + processorSink.detach(); + processorSink = null; + } + } + + private native void setVideoProcessorNative(VideoProcessor processor); + + /** + * Passes a frame the processor delivered on to the tracks. + */ + native void deliverProcessedFrame(VideoFrame frame); + } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoProcessorTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoProcessorTests.java new file mode 100644 index 000000000..577140c15 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoProcessorTests.java @@ -0,0 +1,268 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import dev.onvoid.webrtc.TestBase; +import dev.onvoid.webrtc.media.video.desktop.DesktopSource; +import dev.onvoid.webrtc.media.video.desktop.ScreenCapturer; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +class VideoProcessorTests extends TestBase { + + @Test + void customSourceRejectsProcessor() { + CustomVideoSource source = new CustomVideoSource(); + + try { + assertThrows(UnsupportedOperationException.class, + () -> source.setVideoProcessor(new TestProcessor())); + } + finally { + source.dispose(); + } + } + + @Test + void setAndRemove() { + VideoTrackSource[] sources = { new VideoDesktopSource(), new VideoDeviceSource() }; + + for (VideoTrackSource source : sources) { + TestProcessor processor = new TestProcessor(); + + source.setVideoProcessor(processor); + + assertNotNull(processor.sink, "setSink was not called"); + + source.setVideoProcessor(new TestProcessor()); + source.setVideoProcessor(null); + } + + ((VideoDesktopSource) sources[0]).dispose(); + ((VideoDeviceSource) sources[1]).dispose(); + } + + @Test + void staleSinkDropsFrames() { + VideoDesktopSource source = new VideoDesktopSource(); + TestProcessor first = new TestProcessor(); + TestProcessor second = new TestProcessor(); + + source.setVideoProcessor(first); + source.setVideoProcessor(second); + + VideoFrame frame = new VideoFrame(NativeI420Buffer.allocate(64, 48), 0); + + try { + // The replaced processor's sink, and after disposing both sinks, + // accept frames without passing them on or failing. + first.sink.onVideoFrame(frame); + + source.dispose(); + + first.sink.onVideoFrame(frame); + second.sink.onVideoFrame(frame); + + assertThrows(NullPointerException.class, () -> second.sink.onVideoFrame(null)); + } + finally { + frame.release(); + } + } + + @Test + void processedDesktopFramesReachTrack() throws Exception { + DesktopSource screen = firstScreen(); + + VideoDesktopSource source = new VideoDesktopSource(); + source.setSourceId(screen.id, false); + source.setFrameRate(10); + source.setMaxFrameSize(320, 240); + + VideoTrack track = factory.createVideoTrack("screen", source); + + AtomicInteger processed = new AtomicInteger(); + AtomicReference receivedLuma = new AtomicReference<>(); + AtomicReference size = new AtomicReference<>(); + CountDownLatch received = new CountDownLatch(3); + + // Replaces every frame with a uniformly grey one of the same size. + VideoProcessor processor = new VideoProcessor() { + private VideoTrackSink sink; + + @Override + public void setSink(VideoTrackSink sink) { + this.sink = sink; + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + processed.incrementAndGet(); + + NativeI420Buffer captured = (NativeI420Buffer) frame.buffer; + NativeI420Buffer grey = NativeI420Buffer.allocate(captured.getWidth(), captured.getHeight()); + + fill(grey.getDataY(), (byte) 200); + fill(grey.getDataU(), (byte) 128); + fill(grey.getDataV(), (byte) 128); + + VideoFrame output = new VideoFrame(grey, frame.rotation, frame.timestampNs); + + sink.onVideoFrame(output); + + output.release(); + } + }; + + VideoTrackSink trackSink = frame -> { + NativeI420Buffer buffer = (NativeI420Buffer) frame.buffer; + + receivedLuma.set(buffer.getDataY().get(0) & 0xFF); + size.set(buffer.getWidth() + "x" + buffer.getHeight()); + + frame.release(); + received.countDown(); + }; + + try { + source.setVideoProcessor(processor); + track.addSink(trackSink); + source.start(); + + boolean gotFrames = received.await(10, TimeUnit.SECONDS); + + assumeTrue(gotFrames || processed.get() > 0, "The screen delivered no frames"); + assertTrue(gotFrames, "Processed frames did not reach the track"); + + assertEquals(200, (int) receivedLuma.get()); + assertNotNull(size.get()); + } + finally { + source.stop(); + track.removeSink(trackSink); + track.dispose(); + source.dispose(); + } + } + + @Test + void processorCanDropFrames() throws Exception { + DesktopSource screen = firstScreen(); + + VideoDesktopSource source = new VideoDesktopSource(); + source.setSourceId(screen.id, false); + source.setFrameRate(10); + source.setMaxFrameSize(320, 240); + + VideoTrack track = factory.createVideoTrack("screen", source); + + CountDownLatch processed = new CountDownLatch(3); + AtomicInteger received = new AtomicInteger(); + + VideoTrackSink trackSink = frame -> { + frame.release(); + received.incrementAndGet(); + }; + + try { + source.setVideoProcessor(new VideoProcessor() { + @Override + public void setSink(VideoTrackSink sink) { + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + // Delivers nothing. + processed.countDown(); + } + }); + track.addSink(trackSink); + source.start(); + + assumeTrue(processed.await(10, TimeUnit.SECONDS), "The screen delivered no frames"); + + assertEquals(0, received.get()); + } + finally { + source.stop(); + track.removeSink(trackSink); + track.dispose(); + source.dispose(); + } + } + + private static DesktopSource firstScreen() { + ScreenCapturer capturer = new ScreenCapturer(); + + try { + List screens; + + try { + screens = capturer.getDesktopSources(); + } + catch (Error | RuntimeException e) { + // Without a display, as on headless Linux, listing fails. + screens = null; + } + + assumeFalse(screens == null || screens.isEmpty(), "No screen to capture"); + + return screens.get(0); + } + finally { + capturer.dispose(); + } + } + + private static void fill(ByteBuffer buffer, byte value) { + while (buffer.hasRemaining()) { + buffer.put(value); + } + } + + + + private static class TestProcessor implements VideoProcessor { + + VideoTrackSink sink; + + + @Override + public void setSink(VideoTrackSink sink) { + this.sink = sink; + } + + @Override + public void onFrameCaptured(VideoFrame frame) { + sink.onVideoFrame(frame); + } + } +}