diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp index efb66d9a..734f3577 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp @@ -139,14 +139,21 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } // NV12 has chroma at half the resolution in both directions, and the - // frame is cropped in units of two pixels. - if (settings->width % 2 != 0 || settings->height % 2 != 0) { + // frame is cropped in units of two pixels. The first frame of a source + // can be odd, since WebRTC asks the source for + // requested_resolution_alignment only once the encoder has been set up. + // Refusing it would leave the whole session to the software encoder, so + // the encoder takes the even size below it and drops the last row and + // column of such frames. + if (settings->width < 2 || settings->height < 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } Release(); codecSettings = *settings; + codecSettings.width &= ~1; + codecSettings.height &= ~1; widthInMbs = (codecSettings.width + 15) / 16; heightInMbs = (codecSettings.height + 15) / 16; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; @@ -339,8 +346,12 @@ namespace jni const int height = static_cast(codecSettings.height); // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. - if (i420->width() != width || i420->height() != height) { + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 + && i420->height() - height >= 0 && i420->height() - height <= 1; + + if (!cropped) { webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); scaled->ScaleFrom(*i420); i420 = scaled; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp index 9d36cc30..18f7f572 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvenc/NvencVideoEncoder.cpp @@ -70,14 +70,21 @@ namespace jni if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - // NV12 has chroma at half the resolution in both directions. - if (settings->width % 2 != 0 || settings->height % 2 != 0) { + // NV12 has chroma at half the resolution in both directions. The first + // frame of a source can be odd, since WebRTC asks the source for + // requested_resolution_alignment only once the encoder has been set up. + // Refusing it would leave the whole session to the software encoder, so + // the encoder takes the even size below it and drops the last row and + // column of such frames. + if (settings->width < 2 || settings->height < 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } Release(); codecSettings = *settings; + codecSettings.width &= ~1; + codecSettings.height &= ~1; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); frameCount = 0; @@ -390,8 +397,12 @@ namespace jni const int height = static_cast(codecSettings.height); // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. - if (i420->width() != width || i420->height() != height) { + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 + && i420->height() - height >= 0 && i420->height() - height <= 1; + + if (!cropped) { webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); scaled->ScaleFrom(*i420); i420 = scaled; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp index 13b2d439..ea63224e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoEncoder.cpp @@ -138,8 +138,13 @@ namespace jni if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - // NV12 has chroma at half the resolution in both directions. - if (settings->width % 2 != 0 || settings->height % 2 != 0) { + // NV12 has chroma at half the resolution in both directions. The first + // frame of a source can be odd, since WebRTC asks the source for + // requested_resolution_alignment only once the encoder has been set up. + // Refusing it would leave the whole session to the software encoder, so + // the encoder takes the even size below it and drops the last row and + // column of such frames. + if (settings->width < 2 || settings->height < 2) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } @@ -157,6 +162,8 @@ namespace jni } codecSettings = *settings; + codecSettings.width &= ~1; + codecSettings.height &= ~1; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); @@ -493,8 +500,12 @@ namespace jni const int height = static_cast(codecSettings.height); // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. - if (i420->width() != width || i420->height() != height) { + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 + && i420->height() - height >= 0 && i420->height() - height <= 1; + + if (!cropped) { webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); scaled->ScaleFrom(*i420); i420 = scaled; diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 95d70a06..b4b38c29 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -26,10 +26,13 @@ import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import org.junit.jupiter.api.Test; @@ -112,6 +115,142 @@ void hardwareEncodesAv1() throws Exception { } } + @Test + void hardwareEncodes1080p() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, H264, 1920, 1080)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareEncodesOddSize() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + // NV12 needs even dimensions; the encoder asks for them to be aligned + // and is not left with a size it has to refuse. + try (Run run = new Run(hardware, H264, 641, 361)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + assertEquals(0, run.width.get() % 2, "width " + run.width.get()); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareFollowsResolutionChange() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, H264, 320, 240)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + + // A new size starts the encoder over, with a new session. + run.call.setVideoSize(640, 480); + run.awaitWidth(640); + + run.call.setVideoSize(320, 240); + run.awaitWidth(320); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareFollowsBitrateCap() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, H264, 640, 480)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + + // The encoder is reconfigured in place, without a new session or + // a lost stream. + run.setMaxBitrate(150_000); + run.awaitTargetBitrate(150_000); + run.awaitMoreFrames(10); + + run.setMaxBitrate(2_000_000); + run.awaitMoreFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareRestartsCleanly() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + + try { + // A session or a CUDA context left behind by each call would run + // the GPU out of encoder sessions, and the later calls would be + // encoded in software. + for (int i = 0; i < 15; i++) { + try (Run run = new Run(hardware, H264, 320, 240)) { + run.awaitFrames(5); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + } + } + finally { + hardware.dispose(); + } + } + + @Test + void hardwareEncodesSeveralStreams() throws Exception { + assumeNvenc(); + + PeerConnectionFactory hardware = hardwareFactory(); + List runs = new ArrayList<>(); + + try { + for (int i = 0; i < 4; i++) { + runs.add(new Run(hardware, H264, 320, 240)); + } + for (Run run : runs) { + run.awaitFrames(10); + + assertHardware(run.implementation(), HARDWARE_REQUIRED); + } + } + finally { + for (Run run : runs) { + run.close(); + } + + hardware.dispose(); + } + } + @Test void defaultEncodesH264InSoftware() throws Exception { assumeFalse(OS.contains("mac"), "macOS encodes H.264 through VideoToolbox by default"); @@ -144,6 +283,18 @@ void macEncodesH264WithVideoToolbox() throws Exception { } } + private static void assumeNvenc() { + assumeTrue(OS.contains("win") || OS.contains("linux"), + "hardware encoders are implemented on Windows and Linux only"); + } + + private PeerConnectionFactory hardwareFactory() { + return PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .build(); + } + private static void assertHardware(String implementation, boolean required) { boolean hardwareUsed = implementation.startsWith("NVENC") || implementation.startsWith("VA-API") @@ -203,4 +354,108 @@ private static String encoderImplementation(PeerConnectionFactory factory, } } + /** + * A call that sends video in the given codec and size, and counts what + * the receiver gets. + */ + private static final class Run implements AutoCloseable { + + final TestMediaCall call; + final RTCRtpReceiver receiver; + final VideoTrack track; + final VideoTrackSink sink; + final AtomicInteger frames = new AtomicInteger(); + final AtomicInteger width = new AtomicInteger(); + + + Run(PeerConnectionFactory factory, Predicate codec, int width, + int height) throws Exception { + call = new TestMediaCall(factory, true, false, codec); + call.setVideoSize(width, height); + call.negotiate(); + + receiver = call.getReceiver("video"); + track = (VideoTrack) receiver.getTrack(); + sink = frame -> { + this.width.set(frame.buffer.getWidth()); + frame.release(); + frames.incrementAndGet(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + } + + void awaitFrames(int count) throws InterruptedException { + await(() -> frames.get() >= count, "frames: " + frames.get() + " of " + count); + } + + void awaitMoreFrames(int count) throws InterruptedException { + awaitFrames(frames.get() + count); + } + + void awaitWidth(int expected) throws InterruptedException { + await(() -> width.get() == expected, "width " + width.get() + ", expected " + expected); + } + + void setMaxBitrate(int bitrate) { + RTCRtpSendParameters parameters = call.getVideoSender().getParameters(); + parameters.encodings.get(0).maxBitrate = bitrate; + + call.getVideoSender().setParameters(parameters); + } + + void awaitTargetBitrate(double atMost) throws InterruptedException { + await(() -> { + Map outbound = call.getOutboundVideoStats(); + Object target = outbound == null ? null : outbound.get("targetBitrate"); + + return target instanceof Number && ((Number) target).doubleValue() <= atMost; + }, "target bitrate not at or below " + atMost); + } + + String implementation() throws InterruptedException { + // The statistics catch up with the encoder shortly after. + Object[] implementation = new Object[1]; + + await(() -> { + Map outbound = call.getOutboundVideoStats(); + + implementation[0] = outbound == null ? null : outbound.get("encoderImplementation"); + + return implementation[0] != null; + }, "no encoder implementation in the statistics"); + + return String.valueOf(implementation[0]); + } + + @Override + public void close() throws InterruptedException { + track.removeSink(sink); + receiver.dispose(); + call.close(); + } + + private static void await(Condition condition, String description) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (System.nanoTime() < deadline) { + if (condition.test()) { + return; + } + + Thread.sleep(50); + } + + assertTrue(condition.test(), description); + } + + private interface Condition { + + boolean test() throws InterruptedException; + + } + } + }