From df2d793f52fe866ae6594335ef0aebf3a9c27d6a Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 11 Oct 2026 23:07:41 +0200 Subject: [PATCH] fix: capture screens with ScreenCaptureKit on macOS A VideoDesktopSource capturing a screen on macOS always used ScreenCapturerMac, since WebRTC tries ScreenCaptureKit only when allow_sck_capturer is set. That capturer takes its first frame from a display snapshot and afterwards only copies what its CGDisplayStream reports as changed, so when the stream stops calling back, it keeps sending the first frame, with a live cursor composited on top (#344). Screens are now captured with ScreenCaptureKit on macOS 14 and newer, and with ScreenCapturerMac as before on older releases. ScreenCaptureKit is told the source's frame rate, as it otherwise captures at the display refresh rate. Source lists stay with ScreenCapturerMac: without its system picker ScreenCaptureKit lists no screens, and both name a screen by its display ID. Window capture is unchanged. A new test switches a window of its own between black and white while the screen is captured, and checks the frames follow. --- .../media/video/VideoTrackDesktopSource.cpp | 15 ++ .../media/video/VideoDesktopSourceTests.java | 166 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java diff --git a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp index af0fc356..996d8a51 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp @@ -267,6 +267,15 @@ namespace jni #if defined(WEBRTC_MAC) options.set_allow_iosurface(true); + // Screens are captured with ScreenCaptureKit on macOS 14 and newer. + // The CGDisplayStream capturer otherwise used gets new pixels only + // when its stream calls back, and when that stops it keeps sending the + // first frame with a live cursor on top (issue #344). Older macOS + // falls back to it. Only screens: window capture ignores this, and the + // source list stays with the CGDisplayStream capturer in + // DesktopCapturer, since ScreenCaptureKit lists no screens without its + // system picker. Both name a screen by its CGDirectDisplayID. + options.set_allow_sck_capturer(true); #endif #if defined(WEBRTC_WIN) options.set_allow_directx_capturer(true); @@ -307,6 +316,12 @@ namespace jni return; } +#if defined(WEBRTC_MAC) + // ScreenCaptureKit captures at the display refresh rate unless told + // otherwise, though only one frame per interval below is taken. + capturer->SetMaxFrameRate(frameRate); +#endif + capturer->Start(this); if (focusSelectedSource) { diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java new file mode 100644 index 00000000..86b66c9a --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/VideoDesktopSourceTests.java @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import dev.onvoid.webrtc.TestBase; +import dev.onvoid.webrtc.media.video.desktop.DesktopSource; +import dev.onvoid.webrtc.media.video.desktop.ScreenCapturer; + +import java.awt.Color; +import java.awt.EventQueue; +import java.awt.Frame; +import java.awt.GraphicsEnvironment; +import java.awt.Rectangle; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +@Execution(ExecutionMode.SAME_THREAD) +class VideoDesktopSourceTests extends TestBase { + + private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT); + + @Test + void screenFramesFollowScreen() throws Exception { + assumeTrue(OS.contains("mac"), "covers the macOS screen capturers"); + assumeFalse(GraphicsEnvironment.isHeadless(), "No display"); + + DesktopSource screen = firstScreen(); + + // A window of our own, switched between black and white. A capture + // that froze on its first frame keeps showing one of the two, while + // still delivering frames (issue #344). + Frame window = new Frame(); + Rectangle bounds = GraphicsEnvironment.getLocalGraphicsEnvironment() + .getDefaultScreenDevice().getDefaultConfiguration().getBounds(); + + EventQueue.invokeAndWait(() -> { + window.setUndecorated(true); + window.setAlwaysOnTop(true); + window.setBackground(Color.BLACK); + window.setBounds(bounds.x + bounds.width / 10, bounds.y + bounds.height / 10, + bounds.width * 8 / 10, bounds.height * 8 / 10); + window.setVisible(true); + window.toFront(); + }); + + VideoDesktopSource source = new VideoDesktopSource(); + source.setSourceId(screen.id, false); + source.setFrameRate(10); + source.setMaxFrameSize(320, 240); + + VideoTrack track = factory.createVideoTrack("screen", source); + + List luma = new ArrayList<>(); + + VideoTrackSink sink = frame -> { + NativeI420Buffer buffer = (NativeI420Buffer) frame.buffer; + double mean = meanLuma(buffer); + + frame.release(); + + synchronized (luma) { + luma.add(mean); + } + }; + + try { + track.addSink(sink); + source.start(); + + // Let the capture start on a black window, then turn it white + // and back, a few times. + Thread.sleep(1000); + + for (int i = 0; i < 6; i++) { + Color color = i % 2 == 0 ? Color.WHITE : Color.BLACK; + + EventQueue.invokeAndWait(() -> { + window.setBackground(color); + window.repaint(); + }); + + Thread.sleep(500); + } + } + finally { + source.stop(); + track.removeSink(sink); + track.dispose(); + source.dispose(); + + EventQueue.invokeAndWait(window::dispose); + } + + List means; + + synchronized (luma) { + means = new ArrayList<>(luma); + } + + assumeFalse(means.isEmpty(), "The screen delivered no frames"); + + double min = means.stream().mapToDouble(Double::doubleValue).min().getAsDouble(); + double max = means.stream().mapToDouble(Double::doubleValue).max().getAsDouble(); + + // The window covers about two thirds of the screen, so white and black + // differ by more than 100 in mean luma. + assertTrue(max - min > 50, + "Frames did not follow the screen: " + means.size() + " frames, mean luma " + + String.format(Locale.ROOT, "%.1f to %.1f", min, max)); + } + + private static double meanLuma(NativeI420Buffer buffer) { + ByteBuffer data = buffer.getDataY(); + int stride = buffer.getStrideY(); + int width = buffer.getWidth(); + int height = buffer.getHeight(); + long sum = 0; + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + sum += data.get(y * stride + x) & 0xFF; + } + } + + return (double) sum / (width * height); + } + + private static DesktopSource firstScreen() { + ScreenCapturer capturer = new ScreenCapturer(); + + try { + List screens = capturer.getDesktopSources(); + + assumeFalse(screens == null || screens.isEmpty(), "No screen to capture"); + + return screens.get(0); + } + finally { + capturer.dispose(); + } + } +}