From 1803c0d337afc6a4a02f8d93e217344a78ceb48a Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 4 Oct 2026 12:56:18 +0200 Subject: [PATCH] test: check that hardware decoders crop padded frames to the picture A decoder puts a picture whose size is not a multiple of its block size into a larger surface: 360 rows, for one, sit in a texture of 368 or more. MFVideoDecoder copies the picture out of that surface, taking the chroma plane from below the padded height and cropping to the display aperture, but the tests sent 320x240, which needs no padding, and checked only the size and the first row of luma, so none of that was exercised. The new tests send 640x360 in H.264, AV1 and VP9 and check every decoded frame: the shape that was sent, the luma gradient in the first and the last row, and the chroma. The test call now sends flat chroma (U 90, V 160), so chroma read from the wrong offset, which would be luma or padding, shows. Each test runs where the codec is decoded in hardware and follows the properties that require a hardware decoder. On an AMD Radeon RX 9070 XT the three tests pass. With the chroma plane taken from the picture height instead of the texture's, all three fail, so the decoded textures are padded there and the tests see the difference. --- .../HardwareVideoDecoderIntegrationTest.java | 167 ++++++++++++++++-- .../java/dev/onvoid/webrtc/TestMediaCall.java | 17 ++ 2 files changed, 172 insertions(+), 12 deletions(-) diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index 1774ff03..50926791 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -33,7 +33,9 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Predicate; import org.junit.jupiter.api.Test; @@ -92,6 +94,158 @@ void hardwareDecodesAv1() throws Exception { assertHardwareDecodes(AV1, HARDWARE_AV1_REQUIRED); } + @Test + void hardwareCropsPaddedH264() throws Exception { + assertCropsPaddedFrames(H264, implementation -> expectHardware(implementation, HARDWARE_REQUIRED)); + } + + @Test + void hardwareCropsPaddedAv1() throws Exception { + assertCropsPaddedFrames(AV1, implementation -> expectHardware(implementation, HARDWARE_AV1_REQUIRED)); + } + + @Test + void hardwareCropsPaddedVp9() throws Exception { + assumeVp9Platform(); + + assertCropsPaddedFrames(VP9, HardwareVideoDecoderIntegrationTest::expectVp9Hardware); + } + + /** + * Sends 640x360, a height that is not a multiple of 16 or 64, so the + * decoder puts the picture into a larger surface, and checks that what + * comes out is the picture: the size that was sent (or a scaled-down one + * of the same shape), the luma gradient in the first and the last row, + * and the flat chroma of the call where it should be. Chroma read from the + * wrong offset of the surface would be luma, or padding. + */ + private void assertCropsPaddedFrames(Predicate codec, Consumer expectHardware) + throws Exception { + final int width = 640; + final int height = 360; + + PeerConnectionFactory hardware = hardwareFactory(); + CountDownLatch received = new CountDownLatch(30); + AtomicReference wrong = new AtomicReference<>(); + AtomicInteger padded = new AtomicInteger(); + String implementation; + + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { + call.setVideoSize(width, height); + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + int frameWidth = frame.buffer.getWidth(); + int frameHeight = frame.buffer.getHeight(); + + if (frameWidth * height != frameHeight * width) { + // Not the shape that was sent: padding came along. + wrong.compareAndSet(null, "a frame of " + frameWidth + "x" + frameHeight); + } + else { + String error = checkPicture(frame.buffer.toI420()); + + if (error != null) { + wrong.compareAndSet(null, "a " + frameWidth + "x" + frameHeight + " frame with " + error); + } + if (frameHeight % 16 != 0) { + padded.incrementAndGet(); + } + } + + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + implementation = decoderImplementationOf(call); + + track.removeSink(sink); + receiver.dispose(); + } + finally { + hardware.dispose(); + } + + expectHardware.accept(implementation); + + assertNull(wrong.get(), implementation + " decoded " + wrong.get()); + assertTrue(padded.get() > 0, "no frame needed padding"); + } + + /** + * Returns what is wrong with the picture of a frame the call sent, or + * {@code null}. + */ + private static String checkPicture(I420Buffer buffer) { + ByteBuffer y = buffer.getDataY(); + int lastRow = (buffer.getHeight() - 1) * buffer.getStrideY(); + + if (!isRamp(y, 0, buffer.getWidth())) { + return "no gradient in the first row"; + } + if (!isRamp(y, lastRow, buffer.getWidth())) { + return "no gradient in the last row"; + } + + int chromaWidth = (buffer.getWidth() + 1) / 2; + int chromaHeight = (buffer.getHeight() + 1) / 2; + + if (!isFlat(buffer.getDataU(), buffer.getStrideU(), chromaWidth, chromaHeight, TestMediaCall.CHROMA_U)) { + return "U not at " + TestMediaCall.CHROMA_U; + } + if (!isFlat(buffer.getDataV(), buffer.getStrideV(), chromaWidth, chromaHeight, TestMediaCall.CHROMA_V)) { + return "V not at " + TestMediaCall.CHROMA_V; + } + + return null; + } + + private static boolean isRamp(ByteBuffer plane, int offset, int length) { + int min = 255; + int max = 0; + + for (int x = 0; x < length; x++) { + int value = plane.get(offset + x) & 0xff; + + min = Math.min(min, value); + max = Math.max(max, value); + } + + return max - min > 150; + } + + private static boolean isFlat(ByteBuffer plane, int stride, int width, int height, int expected) { + for (int row = 0; row < height; row++) { + for (int x = 0; x < width; x++) { + int value = plane.get(row * stride + x) & 0xff; + + // Coding leaves a flat plane close to, not exactly at, its value. + if (Math.abs(value - expected) > 16) { + return false; + } + } + } + + return true; + } + + private static void expectHardware(String implementation, boolean required) { + if (required) { + assertTrue(isHardware(implementation), implementation); + } + else { + assumeTrue(isHardware(implementation), "no hardware decoder: " + implementation); + } + } + private void assertHardwareDecodes(Predicate codec, boolean required) throws Exception { assumeTrue(OS.contains("win"), "hardware decoders are implemented on Windows only"); @@ -440,19 +594,8 @@ else if (!hasPicture(frame.buffer.toI420())) { * it is read as it is and stays with the frame. */ private static boolean hasPicture(I420Buffer buffer) { - ByteBuffer y = buffer.getDataY(); - int min = 255; - int max = 0; - // The first row has the whole ramp, and a wrap of it. - for (int x = 0; x < buffer.getWidth(); x++) { - int value = y.get(x) & 0xff; - - min = Math.min(min, value); - max = Math.max(max, value); - } - - return max - min > 150; + return isRamp(buffer.getDataY(), 0, buffer.getWidth()); } /** diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index 118f09f6..149f9e33 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -49,6 +49,12 @@ class TestMediaCall implements AutoCloseable { private static final int DEFAULT_WIDTH = 320; private static final int DEFAULT_HEIGHT = 240; + /** The value of every U sample the call sends. */ + static final int CHROMA_U = 90; + + /** The value of every V sample the call sends. */ + static final int CHROMA_V = 160; + private final CustomVideoSource videoSource; private final CustomAudioSource audioSource; private final VideoTrack videoTrack; @@ -331,6 +337,11 @@ private void feed() { } shade += 3; + // Flat chroma, so that a receiver can tell where the decoder + // took its chroma from. + fill(buffer.getDataU(), CHROMA_U); + fill(buffer.getDataV(), CHROMA_V); + VideoFrame frame = new VideoFrame(buffer, 0); videoSource.pushFrame(frame); frame.release(); @@ -346,4 +357,10 @@ private void feed() { } } + private static void fill(ByteBuffer plane, int value) { + for (int i = 0; i < plane.capacity(); i++) { + plane.put(i, (byte) value); + } + } + }