From 805c6114a2fa3c32048c3627e406b29664092f68 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 4 Oct 2026 17:12:52 +0200 Subject: [PATCH 1/3] fix: keep NVENC for sources whose first frame has an odd size WebRTC asks a source for the encoder's requested_resolution_alignment only after the encoder has been set up, so the first frame of a window capture or a camera with an odd size reaches InitEncode as it is. NvencVideoEncoder refused it with WEBRTC_VIDEO_CODEC_ERR_PARAMETER, and the fallback encoder then left the whole session to OpenH264. The encoder now takes the even size below an odd one and drops the last row and column of such frames, which needs no copy. New tests send 1080p, an odd 641x361, a change of resolution, a bitrate cap that is lifted again, fifteen calls one after the other (a leaked session would run the GPU out of them) and four calls at once, and require NVENC for all of them. The odd size failed before the change. Run on an NVIDIA GeForce RTX 3080 on Windows, also with -Xcheck:jni. --- .../video/codec/nvenc/NvencVideoEncoder.cpp | 19 +- .../HardwareVideoEncoderIntegrationTest.java | 255 ++++++++++++++++++ 2 files changed, 270 insertions(+), 4 deletions(-) diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp index 9d36cc30..18f7f572 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp @@ -70,14 +70,21 @@ namespace jni if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - // NV12 has chroma at half the resolution in both directions. - if (settings->width % 2 != 0 || settings->height % 2 != 0) { + // NV12 has chroma at half the resolution in both directions. The first + // frame of a source can be odd, since WebRTC asks the source for + // requested_resolution_alignment only once the encoder has been set up. + // Refusing it would leave the whole session to the software encoder, so + // the encoder takes the even size below it and drops the last row and + // column of such frames. + if (settings->width < 2 || settings->height < 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } Release(); codecSettings = *settings; + codecSettings.width &= ~1; + codecSettings.height &= ~1; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); frameCount = 0; @@ -390,8 +397,12 @@ namespace jni const int height = static_cast(codecSettings.height); // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. - if (i420->width() != width || i420->height() != height) { + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 + && i420->height() - height >= 0 && i420->height() - height <= 1; + + if (!cropped) { webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); scaled->ScaleFrom(*i420); i420 = scaled; diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 95d70a06..b4b38c29 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -26,10 +26,13 @@ import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import org.junit.jupiter.api.Test; @@ -112,6 +115,142 @@ void hardwareEncodesAv1() throws Exception { } } + @Test + void hardwareEncodes1080p() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, H264, 1920, 1080)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareEncodesOddSize() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + // NV12 needs even dimensions; the encoder asks for them to be aligned + // and is not left with a size it has to refuse. + try (Run run = new Run(hardware, H264, 641, 361)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + assertEquals(0, run.width.get() % 2, "width " + run.width.get()); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareFollowsResolutionChange() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, H264, 320, 240)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + + // A new size starts the encoder over, with a new session. + run.call.setVideoSize(640, 480); + run.awaitWidth(640); + + run.call.setVideoSize(320, 240); + run.awaitWidth(320); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareFollowsBitrateCap() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, H264, 640, 480)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + + // The encoder is reconfigured in place, without a new session or + // a lost stream. + run.setMaxBitrate(150_000); + run.awaitTargetBitrate(150_000); + run.awaitMoreFrames(10); + + run.setMaxBitrate(2_000_000); + run.awaitMoreFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareRestartsCleanly() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try { + // A session or a CUDA context left behind by each call would run + // the GPU out of encoder sessions, and the later calls would be + // encoded in software. + for (int i = 0; i < 15; i++) { + try (Run run = new Run(hardware, H264, 320, 240)) { + run.awaitFrames(5); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + } + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareEncodesSeveralStreams() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + List runs = new ArrayList<>(); + + try { + for (int i = 0; i < 4; i++) { + runs.add(new Run(hardware, H264, 320, 240)); + } + for (Run run : runs) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + } + finally { + for (Run run : runs) { + run.close(); + } + + hardware.dispose(); + } + } + @Test void defaultEncodesH264InSoftware() throws Exception { assumeFalse(OS.contains("mac"), "macOS encodes H.264 through VideoToolbox by default"); @@ -144,6 +283,18 @@ void macEncodesH264WithVideoToolbox() throws Exception { } } + private static void assumeNvenc() { + assumeTrue(OS.contains("win") || OS.contains("linux"), + "hardware encoders are implemented on Windows and Linux only"); + } + + private PeerConnectionFactory hardwareFactory() { + return PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .build(); + } + private static void assertHardware(String implementation, boolean required) { boolean hardwareUsed = implementation.startsWith("NVENC") || implementation.startsWith("VA-API") @@ -203,4 +354,108 @@ private static String encoderImplementation(PeerConnectionFactory factory, } } + /** + * A call that sends video in the given codec and size, and counts what + * the receiver gets. + */ + private static final class Run implements AutoCloseable { + + final TestMediaCall call; + final RTCRtpReceiver receiver; + final VideoTrack track; + final VideoTrackSink sink; + final AtomicInteger frames = new AtomicInteger(); + final AtomicInteger width = new AtomicInteger(); + + + Run(PeerConnectionFactory factory, Predicate codec, int width, + int height) throws Exception { + call = new TestMediaCall(factory, true, false, codec); + call.setVideoSize(width, height); + call.negotiate(); + + receiver = call.getReceiver("video"); + track = (VideoTrack) receiver.getTrack(); + sink = frame -> { + this.width.set(frame.buffer.getWidth()); + frame.release(); + frames.incrementAndGet(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + } + + void awaitFrames(int count) throws InterruptedException { + await(() -> frames.get() >= count, "frames: " + frames.get() + " of " + count); + } + + void awaitMoreFrames(int count) throws InterruptedException { + awaitFrames(frames.get() + count); + } + + void awaitWidth(int expected) throws InterruptedException { + await(() -> width.get() == expected, "width " + width.get() + ", expected " + expected); + } + + void setMaxBitrate(int bitrate) { + RTCRtpSendParameters parameters = call.getVideoSender().getParameters(); + parameters.encodings.get(0).maxBitrate = bitrate; + + call.getVideoSender().setParameters(parameters); + } + + void awaitTargetBitrate(double atMost) throws InterruptedException { + await(() -> { + Map outbound = call.getOutboundVideoStats(); + Object target = outbound == null ? null : outbound.get("targetBitrate"); + + return target instanceof Number && ((Number) target).doubleValue() <= atMost; + }, "target bitrate not at or below " + atMost); + } + + String implementation() throws InterruptedException { + // The statistics catch up with the encoder shortly after. + Object[] implementation = new Object[1]; + + await(() -> { + Map outbound = call.getOutboundVideoStats(); + + implementation[0] = outbound == null ? null : outbound.get("encoderImplementation"); + + return implementation[0] != null; + }, "no encoder implementation in the statistics"); + + return String.valueOf(implementation[0]); + } + + @Override + public void close() throws InterruptedException { + track.removeSink(sink); + receiver.dispose(); + call.close(); + } + + private static void await(Condition condition, String description) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (System.nanoTime() < deadline) { + if (condition.test()) { + return; + } + + Thread.sleep(50); + } + + assertTrue(condition.test(), description); + } + + private interface Condition { + + boolean test() throws InterruptedException; + + } + } + } From 829538d0ca4c20806586fc46b9dca8ac2b16ec78 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 4 Oct 2026 18:48:57 +0200 Subject: [PATCH 2/3] fix: keep Media Foundation for sources whose first frame has an odd size The same as for NVENC: WebRTC asks a source for the encoder's requested_resolution_alignment only after the encoder has been set up, so an odd first frame reached MFVideoEncoder::InitEncode, which refused it with WEBRTC_VIDEO_CODEC_ERR_PARAMETER, and the fallback encoder left the whole session to OpenH264. The encoder now takes the even size below an odd one and drops the last row and column of such frames, which needs no copy. The input type and the encoded size follow the even size. Compiled, and the NVENC path next to it is unchanged (all tests of HardwareVideoEncoderIntegrationTest pass), but not run with a Media Foundation encoder: the only H.264 one on the machine it was written on is a leftover AMD driver component that cannot start without the GPU. hardwareEncodesOddSize covers it wherever Media Foundation is the first hardware encoder. --- .../video/codec/windows/MFVideoEncoder.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp index 13b2d439..ea63224e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp @@ -138,8 +138,13 @@ namespace jni if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - // NV12 has chroma at half the resolution in both directions. - if (settings->width % 2 != 0 || settings->height % 2 != 0) { + // NV12 has chroma at half the resolution in both directions. The first + // frame of a source can be odd, since WebRTC asks the source for + // requested_resolution_alignment only once the encoder has been set up. + // Refusing it would leave the whole session to the software encoder, so + // the encoder takes the even size below it and drops the last row and + // column of such frames. + if (settings->width < 2 || settings->height < 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } @@ -157,6 +162,8 @@ namespace jni } codecSettings = *settings; + codecSettings.width &= ~1; + codecSettings.height &= ~1; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); @@ -493,8 +500,12 @@ namespace jni const int height = static_cast(codecSettings.height); // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. - if (i420->width() != width || i420->height() != height) { + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 + && i420->height() - height >= 0 && i420->height() - height <= 1; + + if (!cropped) { webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); scaled->ScaleFrom(*i420); i420 = scaled; From f40a08478f17d4f498e55108a1e3ee67ccf22e6b Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 4 Oct 2026 19:02:37 +0200 Subject: [PATCH 3/3] fix: keep VA-API for sources whose first frame has an odd size The same as for NVENC and Media Foundation: WebRTC asks a source for the encoder's requested_resolution_alignment only after the encoder has been set up, so an odd first frame reached VaapiH264Encoder::InitEncode, which refused it with WEBRTC_VIDEO_CODEC_ERR_PARAMETER, and the fallback encoder left the whole session to OpenH264. The encoder now takes the even size below an odd one and drops the last row and column of such frames, which needs no copy. The macroblock counts, the cropping in the sequence parameters and the encoded size follow the even size, which the cropping (in units of two samples) needs anyway. Neither compiled nor run: this machine is Windows without a Linux toolchain or WSL, and has no VA-API device. The change is the one made to NVENC, which was built and tested, and CI compiles this file on the Linux builds. hardwareEncodesOddSize covers it on a machine where VA-API is the first hardware encoder. --- .../video/codec/linux/VaapiH264Encoder.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp index efb66d9a..734f3577 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp @@ -139,14 +139,21 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } // NV12 has chroma at half the resolution in both directions, and the - // frame is cropped in units of two pixels. - if (settings->width % 2 != 0 || settings->height % 2 != 0) { + // frame is cropped in units of two pixels. The first frame of a source + // can be odd, since WebRTC asks the source for + // requested_resolution_alignment only once the encoder has been set up. + // Refusing it would leave the whole session to the software encoder, so + // the encoder takes the even size below it and drops the last row and + // column of such frames. + if (settings->width < 2 || settings->height < 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } Release(); codecSettings = *settings; + codecSettings.width &= ~1; + codecSettings.height &= ~1; widthInMbs = (codecSettings.width + 15) / 16; heightInMbs = (codecSettings.height + 15) / 16; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; @@ -339,8 +346,12 @@ namespace jni const int height = static_cast(codecSettings.height); // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. - if (i420->width() != width || i420->height() != height) { + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 + && i420->height() - height >= 0 && i420->height() - height <= 1; + + if (!cropped) { webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); scaled->ScaleFrom(*i420); i420 = scaled;