From cb8b6df9355acfc2aae2e2513af6c488dc2725b3 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 2 Oct 2026 08:42:07 +0200 Subject: [PATCH] feat: decode VP9 on the GPU with Media Foundation on Windows HardwareVideoDecoderFactory now decodes VP9 profile 0 on Windows, through the VP9 decoder Media Foundation has on Direct3D 11, the way it does H.264 and AV1. The factory offers VP9 where the GPU has the DXVA decoder profile (D3D11_DECODER_PROFILE_VP9_VLD_PROFILE0, with NV12 output) and Windows has a VP9 decoder that uses Direct3D 11: the VP9 Video Extensions of the Microsoft Store. Without either, VP9 stays with libvpx, and negotiation does not change. MFVideoDecoder takes VP9 as a third codec. Two things are particular to it: - A frame with spatial layers goes to libvpx. Its layers reach a decoder back to back without a superframe index; VideoToolbox does not decode that, and a Media Foundation decoder is not known to. - A VP9 stream states its size in the key frames only, so the decoder may have no output type to offer before it has seen one. That is no longer a failure to configure; the output type is set when the transform asks for it, as it is for a change of size. The VP9 tests of the decoder test class run on Windows too, not on macOS only. A VP9 decoder is required with -Dwebrtc.test.hardwareVp9Decoder=true on Windows, as an AV1 decoder is with its own property; on macOS the existing property applies. This has not been built or run on Windows. --- docs/guide/advanced/video-codecs.md | 4 +- .../video/codec/windows/MFVideoDecoder.h | 4 +- .../codec/windows/MFVideoDecoderFactory.h | 12 ++- .../video/codec/windows/MFVideoDecoder.cpp | 28 +++++- .../codec/windows/MFVideoDecoderFactory.cpp | 18 +++- .../codec/HardwareVideoDecoderFactory.java | 7 +- .../HardwareVideoDecoderIntegrationTest.java | 91 ++++++++++++------- 7 files changed, 112 insertions(+), 52 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index c8cb8a55..2d759ea5 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -51,11 +51,11 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| -| Windows | H.264 and AV1, on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | +| Windows | H.264, AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | | Linux | Not yet; decoding is in software | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`, and VP9 (profile 0) through VideoToolbox's VP9 decoder, on Macs that have one | -AV1 on Windows needs the *AV1 Video Extension*, which Windows 11 includes. Decoded frames are copied from GPU memory back to system memory, where WebRTC's frames are, so hardware decoding pays off mostly at high resolutions and with many streams; at low resolutions WebRTC's software decoders are about as cheap. A hardware decoder that fails, or turns out to decode in software, is replaced by the software decoder, which starts with the next key frame. +AV1 on Windows needs the *AV1 Video Extension*, which Windows 11 includes, and VP9 the *VP9 Video Extensions*, a free component of the Microsoft Store that a system may not have; without it, VP9 is decoded in software. Streams with spatial layers (SVC) are decoded in software too, since their layers reach a decoder without the index that says where each ends. Decoded frames are copied from GPU memory back to system memory, where WebRTC's frames are, so hardware decoding pays off mostly at high resolutions and with many streams; at low resolutions WebRTC's software decoders are about as cheap. A hardware decoder that fails, or turns out to decode in software, is replaced by the software decoder, which starts with the next key frame. On macOS, VP9 decoding in hardware saves CPU: in a measurement on an Apple M2 it took a fraction of the processor time libvpx needs, but multi-threaded libvpx was about as fast on the clock. VideoToolbox decodes VP9 in hardware only where the Mac has a decoder for it, which Apple silicon does; a Mac without one decodes VP9 with libvpx as before. Profile 2 (10 bit), sizes below 64x64 or above 4096x4096, and streams with spatial layers (SVC) are decoded with libvpx too. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h index 55ae783b..6321a901 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h @@ -38,7 +38,7 @@ namespace jni { - // Decodes H.264 or AV1 on the GPU, with the decoder transform of Windows + // Decodes H.264, AV1 or VP9 on the GPU, with the decoder transform of Windows // for the codec, which decodes through DXVA on the Direct3D 11 device it // is given. // @@ -52,7 +52,7 @@ namespace jni class MFVideoDecoder : public webrtc::VideoDecoder { public: - // The codec is H.264 or AV1. + // The codec is H.264, AV1 or VP9. explicit MFVideoDecoder(webrtc::VideoCodecType codec); ~MFVideoDecoder() override; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h index 5dfe2640..13e5248b 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h @@ -27,14 +27,15 @@ namespace jni { - // Creates the Media Foundation decoders that decode on the GPU, for H.264 - // and AV1, whichever the GPU decodes and Windows has a decoder for that + // Creates the Media Foundation decoders that decode on the GPU, for H.264, + // AV1 and VP9, whichever the GPU decodes and Windows has a decoder for that // uses Direct3D 11. It offers them in the formats WebRTC's software - // decoders offer too: H.264 in all its profiles, and AV1 in profile 0. + // decoders offer too: H.264 in all its profiles, and AV1 and VP9 in + // profile 0. class MFVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: - // Returns a factory, or null if the GPU decodes neither codec. + // Returns a factory, or null if the GPU decodes none of the codecs. static std::unique_ptr Create(); ~MFVideoDecoderFactory() override = default; @@ -44,11 +45,12 @@ namespace jni const webrtc::SdpVideoFormat & format) override; private: - MFVideoDecoderFactory(bool h264, bool av1); + MFVideoDecoderFactory(bool h264, bool av1, bool vp9); private: const bool h264; const bool av1; + const bool vp9; }; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp index 2bbd88fc..3d3c7657 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp @@ -56,7 +56,14 @@ namespace jni const GUID & InputFormat(webrtc::VideoCodecType codec) { - return codec == webrtc::kVideoCodecAV1 ? MFVideoFormat_AV1 : MFVideoFormat_H264; + switch (codec) { + case webrtc::kVideoCodecAV1: + return MFVideoFormat_AV1; + case webrtc::kVideoCodecVP9: + return MFVideoFormat_VP90; + default: + return MFVideoFormat_H264; + } } } @@ -174,6 +181,16 @@ namespace jni if (SUCCEEDED(hr)) { hr = SetOutputType(); + + // A VP9 stream states its size in the key frames only, and its + // decoder may offer no output type before it has seen one. It then + // asks for the type again with MF_E_TRANSFORM_STREAM_CHANGE, which + // DrainOutput answers, and a decoder that cannot give one at all + // fails there, and the software decoder takes over. + if (FAILED(hr) && codec == webrtc::kVideoCodecVP9 && !resolution.Valid()) { + RTC_LOG(LS_INFO) << implementationName << " offers no output type yet, hr=" << hr; + hr = S_OK; + } } if (SUCCEEDED(hr)) { hr = transform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); @@ -241,6 +258,15 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + // The layers of a VP9 frame with spatial layers reach a decoder back to + // back, without the superframe index that tells where one ends. libvpx + // takes them that way; a hardware decoder is not known to. + if (codec == webrtc::kVideoCodecVP9 + && (image.SpatialIndex().value_or(0) > 0 || image.SpatialLayerFrameSize(1).has_value())) + { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + ComPtr buffer; HRESULT hr = MFCreateMemoryBuffer(static_cast(image.size()), &buffer); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp index 4f6defed..ad4db2d3 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp @@ -58,6 +58,7 @@ namespace jni { bool h264 = false; bool av1 = false; + bool vp9 = false; try { ComInitializer comInitializer; @@ -86,23 +87,27 @@ namespace jni && HasDirect3DDecoder(MFVideoFormat_H264, manager.Get()); av1 = SupportsDecoderProfile(device.Get(), D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0) && HasDirect3DDecoder(MFVideoFormat_AV1, manager.Get()); + vp9 = SupportsDecoderProfile(device.Get(), D3D11_DECODER_PROFILE_VP9_VLD_PROFILE0) + && HasDirect3DDecoder(MFVideoFormat_VP90, manager.Get()); } catch (...) { return nullptr; } - RTC_LOG(LS_INFO) << "Media Foundation hardware decoders, H.264: " << h264 << ", AV1: " << av1; + RTC_LOG(LS_INFO) << "Media Foundation hardware decoders, H.264: " << h264 << ", AV1: " << av1 + << ", VP9: " << vp9; - if (!h264 && !av1) { + if (!h264 && !av1 && !vp9) { return nullptr; } - return std::unique_ptr(new MFVideoDecoderFactory(h264, av1)); + return std::unique_ptr(new MFVideoDecoderFactory(h264, av1, vp9)); } - MFVideoDecoderFactory::MFVideoDecoderFactory(bool h264, bool av1) : + MFVideoDecoderFactory::MFVideoDecoderFactory(bool h264, bool av1, bool vp9) : h264(h264), - av1(av1) + av1(av1), + vp9(vp9) { } @@ -116,6 +121,9 @@ namespace jni if (av1) { formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); } + if (vp9) { + formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); + } return formats; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 63cabaea..458cb4cf 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -34,8 +34,11 @@ * .build(); * } *

- * On Windows, H.264 and AV1 are decoded on the GPU through Direct3D 11 by the - * Media Foundation decoders of Windows, where the GPU decodes the codec. + * On Windows, H.264, AV1 and VP9 (profile 0) are decoded on the GPU through + * Direct3D 11 by the Media Foundation decoders of Windows, where the GPU + * decodes the codec and Windows has a decoder for it; VP9 needs the VP9 Video + * Extensions of the Microsoft Store. VP9 streams with spatial layers go to + * libvpx. * Decoded frames are copied back to system memory, which WebRTC's frames are * in, so hardware decoding saves CPU mostly at high resolutions. A hardware * decoder that fails to start, or fails while decoding, is replaced by the diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index 91523e76..1774ff03 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -46,7 +46,9 @@ * A machine without a hardware decoder skips the tests that need one, as CI * runners do. Set the system property {@code webrtc.test.hardwareDecoder} to * {@code true} on a machine that has one, to make those tests fail instead, - * and {@code webrtc.test.hardwareAv1Decoder} for a GPU that decodes AV1. + * and {@code webrtc.test.hardwareAv1Decoder} for a GPU that decodes AV1, and + * {@code webrtc.test.hardwareVp9Decoder} for one that decodes VP9 on Windows. + * On macOS the VP9 tests follow {@code webrtc.test.hardwareDecoder}. */ @Execution(ExecutionMode.SAME_THREAD) class HardwareVideoDecoderIntegrationTest extends TestBase { @@ -57,6 +59,8 @@ class HardwareVideoDecoderIntegrationTest extends TestBase { private static final boolean HARDWARE_AV1_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareAv1Decoder"); + private static final boolean HARDWARE_VP9_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareVp9Decoder"); + private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT); private static final Predicate H264 = codec -> @@ -138,14 +142,10 @@ void macDecodesH264WithVideoToolbox() throws Exception { } @Test - void macDecodesVp9WithVideoToolbox() throws Exception { - assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); - - PeerConnectionFactory hardware = PeerConnectionFactory.builder() - .setAudioDeviceModule(audioDevModule) - .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) - .build(); + void hardwareDecodesVp9() throws Exception { + assumeVp9Platform(); + PeerConnectionFactory hardware = hardwareFactory(); String implementation; try { @@ -155,29 +155,22 @@ void macDecodesVp9WithVideoToolbox() throws Exception { hardware.dispose(); } - boolean hardwareUsed = implementation.contains("VideoToolbox"); - - if (HARDWARE_REQUIRED) { - assertTrue(hardwareUsed, implementation); - } - else { - assumeTrue(hardwareUsed, "no hardware decoder: " + implementation); - } + expectVp9Hardware(implementation); } @Test - void macDecodesVp9InSoftwareByDefault() throws Exception { - assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + void defaultDecodesVp9InSoftware() throws Exception { + assumeVp9Platform(); // Hardware decoding is opt-in; the shared factory decodes VP9 with libvpx. String implementation = decoderImplementation(factory, VP9); - assertFalse(implementation.contains("VideoToolbox"), implementation); + assertFalse(isHardware(implementation), implementation); } @Test - void macFollowsResolutionChange() throws Exception { - assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + void hardwareFollowsResolutionChange() throws Exception { + assumeVp9Platform(); PeerConnectionFactory hardware = PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule) @@ -213,7 +206,7 @@ else if (width == 160 && frame.buffer.getHeight() == 120) { String implementation = decoderImplementationOf(call); - assumeTrue(implementation.contains("VideoToolbox"), "no hardware decoder: " + implementation); + expectVp9Hardware(implementation); // The sender restarts at half the size, with a key frame; the // decoder has to start a session for it. @@ -229,7 +222,7 @@ else if (width == 160 && frame.buffer.getHeight() == 120) { assertTrue(half.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "no frames at the new size"); // Still the hardware decoder, not a fallback. - assertTrue(decoderImplementationOf(call).contains("VideoToolbox")); + assertTrue(isHardware(decoderImplementationOf(call))); track.removeSink(sink); receiver.dispose(); @@ -240,12 +233,12 @@ else if (width == 160 && frame.buffer.getHeight() == 120) { } @Test - void macVp9NeedsNoKeyFrames() throws Exception { - assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + void vp9NeedsNoKeyFrames() throws Exception { + assumeVp9Platform(); CallResult result = receiveVideo(hardwareFactory(), VP9, 320, 240, null, 60); - assumeTrue(result.decoder.contains("VideoToolbox"), "no hardware decoder: " + result.decoder); + expectVp9Hardware(result.decoder); // An inter frame that fails to decode makes the receiver ask for a key // frame, and every key frame is then the only frame that decodes. A @@ -255,20 +248,20 @@ void macVp9NeedsNoKeyFrames() throws Exception { } @Test - void macVp9TemporalLayers() throws Exception { - assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + void vp9TemporalLayers() throws Exception { + assumeVp9Platform(); - // Temporal layers are one spatial layer, which VideoToolbox decodes. + // Temporal layers are one spatial layer, which the hardware decodes. CallResult result = receiveVideo(hardwareFactory(), VP9, 320, 240, "L1T3", 60); - assumeTrue(result.decoder.contains("VideoToolbox"), "no hardware decoder: " + result.decoder); + expectVp9Hardware(result.decoder); assertTrue(result.pliCount <= 1, "key frames requested: " + result.pliCount); } @Test - void macVp9SpatialLayers() throws Exception { - assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only"); + void vp9SpatialLayers() throws Exception { + assumeVp9Platform(); // WebRTC encodes spatial layers only above a size, and drops them // below it. @@ -276,9 +269,10 @@ void macVp9SpatialLayers() throws Exception { assumeTrue(result.scalability.startsWith("L2"), "no spatial layers were encoded: " + result.scalability); - // The frames arrive, from libvpx: VideoToolbox does not decode a - // frame whose layers come without a superframe index. - assertFalse(result.decoder.contains("VideoToolbox"), result.decoder); + // The frames arrive, from libvpx: the layers of a frame come without a + // superframe index, which VideoToolbox does not decode, and which a + // Media Foundation decoder is not known to. + assertFalse(isHardware(result.decoder), result.decoder); } private PeerConnectionFactory hardwareFactory() { @@ -288,6 +282,33 @@ private PeerConnectionFactory hardwareFactory() { .build(); } + /** + * Skips the test where VP9 is not decoded in hardware: macOS and Windows. + */ + private static void assumeVp9Platform() { + assumeTrue(OS.contains("mac") || OS.contains("win"), + "VP9 is decoded in hardware on macOS and Windows only"); + } + + private static boolean isHardware(String implementation) { + return implementation.contains("VideoToolbox") || implementation.contains("MediaFoundation"); + } + + /** + * Where a hardware decoder is required, the test fails without one; + * elsewhere it is skipped. + */ + private static void expectVp9Hardware(String implementation) { + boolean required = OS.contains("mac") ? HARDWARE_REQUIRED : HARDWARE_VP9_REQUIRED; + + if (required) { + assertTrue(isHardware(implementation), implementation); + } + else { + assumeTrue(isHardware(implementation), "no hardware decoder: " + implementation); + } + } + /** * What a call that received video tells about its decoder. */