diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index 1774ff03..50926791 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -33,7 +33,9 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Predicate; import org.junit.jupiter.api.Test; @@ -92,6 +94,158 @@ void hardwareDecodesAv1() throws Exception { assertHardwareDecodes(AV1, HARDWARE_AV1_REQUIRED); } + @Test + void hardwareCropsPaddedH264() throws Exception { + assertCropsPaddedFrames(H264, implementation -> expectHardware(implementation, HARDWARE_REQUIRED)); + } + + @Test + void hardwareCropsPaddedAv1() throws Exception { + assertCropsPaddedFrames(AV1, implementation -> expectHardware(implementation, HARDWARE_AV1_REQUIRED)); + } + + @Test + void hardwareCropsPaddedVp9() throws Exception { + assumeVp9Platform(); + + assertCropsPaddedFrames(VP9, HardwareVideoDecoderIntegrationTest::expectVp9Hardware); + } + + /** + * Sends 640x360, a height that is not a multiple of 16 or 64, so the + * decoder puts the picture into a larger surface, and checks that what + * comes out is the picture: the size that was sent (or a scaled-down one + * of the same shape), the luma gradient in the first and the last row, + * and the flat chroma of the call where it should be. Chroma read from the + * wrong offset of the surface would be luma, or padding. + */ + private void assertCropsPaddedFrames(Predicate codec, Consumer expectHardware) + throws Exception { + final int width = 640; + final int height = 360; + + PeerConnectionFactory hardware = hardwareFactory(); + CountDownLatch received = new CountDownLatch(30); + AtomicReference wrong = new AtomicReference<>(); + AtomicInteger padded = new AtomicInteger(); + String implementation; + + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { + call.setVideoSize(width, height); + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + int frameWidth = frame.buffer.getWidth(); + int frameHeight = frame.buffer.getHeight(); + + if (frameWidth * height != frameHeight * width) { + // Not the shape that was sent: padding came along. + wrong.compareAndSet(null, "a frame of " + frameWidth + "x" + frameHeight); + } + else { + String error = checkPicture(frame.buffer.toI420()); + + if (error != null) { + wrong.compareAndSet(null, "a " + frameWidth + "x" + frameHeight + " frame with " + error); + } + if (frameHeight % 16 != 0) { + padded.incrementAndGet(); + } + } + + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + implementation = decoderImplementationOf(call); + + track.removeSink(sink); + receiver.dispose(); + } + finally { + hardware.dispose(); + } + + expectHardware.accept(implementation); + + assertNull(wrong.get(), implementation + " decoded " + wrong.get()); + assertTrue(padded.get() > 0, "no frame needed padding"); + } + + /** + * Returns what is wrong with the picture of a frame the call sent, or + * {@code null}. + */ + private static String checkPicture(I420Buffer buffer) { + ByteBuffer y = buffer.getDataY(); + int lastRow = (buffer.getHeight() - 1) * buffer.getStrideY(); + + if (!isRamp(y, 0, buffer.getWidth())) { + return "no gradient in the first row"; + } + if (!isRamp(y, lastRow, buffer.getWidth())) { + return "no gradient in the last row"; + } + + int chromaWidth = (buffer.getWidth() + 1) / 2; + int chromaHeight = (buffer.getHeight() + 1) / 2; + + if (!isFlat(buffer.getDataU(), buffer.getStrideU(), chromaWidth, chromaHeight, TestMediaCall.CHROMA_U)) { + return "U not at " + TestMediaCall.CHROMA_U; + } + if (!isFlat(buffer.getDataV(), buffer.getStrideV(), chromaWidth, chromaHeight, TestMediaCall.CHROMA_V)) { + return "V not at " + TestMediaCall.CHROMA_V; + } + + return null; + } + + private static boolean isRamp(ByteBuffer plane, int offset, int length) { + int min = 255; + int max = 0; + + for (int x = 0; x < length; x++) { + int value = plane.get(offset + x) & 0xff; + + min = Math.min(min, value); + max = Math.max(max, value); + } + + return max - min > 150; + } + + private static boolean isFlat(ByteBuffer plane, int stride, int width, int height, int expected) { + for (int row = 0; row < height; row++) { + for (int x = 0; x < width; x++) { + int value = plane.get(row * stride + x) & 0xff; + + // Coding leaves a flat plane close to, not exactly at, its value. + if (Math.abs(value - expected) > 16) { + return false; + } + } + } + + return true; + } + + private static void expectHardware(String implementation, boolean required) { + if (required) { + assertTrue(isHardware(implementation), implementation); + } + else { + assumeTrue(isHardware(implementation), "no hardware decoder: " + implementation); + } + } + private void assertHardwareDecodes(Predicate codec, boolean required) throws Exception { assumeTrue(OS.contains("win"), "hardware decoders are implemented on Windows only"); @@ -440,19 +594,8 @@ else if (!hasPicture(frame.buffer.toI420())) { * it is read as it is and stays with the frame. */ private static boolean hasPicture(I420Buffer buffer) { - ByteBuffer y = buffer.getDataY(); - int min = 255; - int max = 0; - // The first row has the whole ramp, and a wrap of it. - for (int x = 0; x < buffer.getWidth(); x++) { - int value = y.get(x) & 0xff; - - min = Math.min(min, value); - max = Math.max(max, value); - } - - return max - min > 150; + return isRamp(buffer.getDataY(), 0, buffer.getWidth()); } /** diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index 118f09f6..149f9e33 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -49,6 +49,12 @@ class TestMediaCall implements AutoCloseable { private static final int DEFAULT_WIDTH = 320; private static final int DEFAULT_HEIGHT = 240; + /** The value of every U sample the call sends. */ + static final int CHROMA_U = 90; + + /** The value of every V sample the call sends. */ + static final int CHROMA_V = 160; + private final CustomVideoSource videoSource; private final CustomAudioSource audioSource; private final VideoTrack videoTrack; @@ -331,6 +337,11 @@ private void feed() { } shade += 3; + // Flat chroma, so that a receiver can tell where the decoder + // took its chroma from. + fill(buffer.getDataU(), CHROMA_U); + fill(buffer.getDataV(), CHROMA_V); + VideoFrame frame = new VideoFrame(buffer, 0); videoSource.pushFrame(frame); frame.release(); @@ -346,4 +357,10 @@ private void feed() { } } + private static void fill(ByteBuffer plane, int value) { + for (int i = 0; i < plane.capacity(); i++) { + plane.put(i, (byte) value); + } + } + }