feat: decode VP9 on the GPU with Media Foundation on Windows

docs/guide/advanced/video-codecs.md

@@ -51,11 +51,11 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
 
 | Platform | Hardware decoding |
 |---|---|
-| Windows | H.264 and AV1, on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) |
+| Windows | H.264, AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) |
 | Linux | Not yet; decoding is in software |
 | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`, and VP9 (profile 0) through VideoToolbox's VP9 decoder, on Macs that have one |
 
-AV1 on Windows needs the *AV1 Video Extension*, which Windows 11 includes. Decoded frames are copied from GPU memory back to system memory, where WebRTC's frames are, so hardware decoding pays off mostly at high resolutions and with many streams; at low resolutions WebRTC's software decoders are about as cheap. A hardware decoder that fails, or turns out to decode in software, is replaced by the software decoder, which starts with the next key frame.
+AV1 on Windows needs the *AV1 Video Extension*, which Windows 11 includes, and VP9 the *VP9 Video Extensions*, a free component of the Microsoft Store that a system may not have; without it, VP9 is decoded in software. Streams with spatial layers (SVC) are decoded in software too, since their layers reach a decoder without the index that says where each ends. Decoded frames are copied from GPU memory back to system memory, where WebRTC's frames are, so hardware decoding pays off mostly at high resolutions and with many streams; at low resolutions WebRTC's software decoders are about as cheap. A hardware decoder that fails, or turns out to decode in software, is replaced by the software decoder, which starts with the next key frame.
 
 On macOS, VP9 decoding in hardware saves CPU: in a measurement on an Apple M2 it took a fraction of the processor time libvpx needs, but multi-threaded libvpx was about as fast on the clock. VideoToolbox decodes VP9 in hardware only where the Mac has a decoder for it, which Apple silicon does; a Mac without one decodes VP9 with libvpx as before. Profile 2 (10 bit), sizes below 64x64 or above 4096x4096, and streams with spatial layers (SVC) are decoded with libvpx too.
 

webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoder.h

@@ -38,7 +38,7 @@
 
 namespace jni
 {
-	// Decodes H.264 or AV1 on the GPU, with the decoder transform of Windows
+	// Decodes H.264, AV1 or VP9 on the GPU, with the decoder transform of Windows
 	// for the codec, which decodes through DXVA on the Direct3D 11 device it
 	// is given.
 	//
@@ -52,7 +52,7 @@ namespace jni
 	class MFVideoDecoder : public webrtc::VideoDecoder
 	{
 		public:
-			// The codec is H.264 or AV1.
+			// The codec is H.264, AV1 or VP9.
 			explicit MFVideoDecoder(webrtc::VideoCodecType codec);
 			~MFVideoDecoder() override;
 

webrtc-jni/src/main/cpp/include/media/video/codec/windows/MFVideoDecoderFactory.h

@@ -27,14 +27,15 @@
 
 namespace jni
 {
-	// Creates the Media Foundation decoders that decode on the GPU, for H.264
-	// and AV1, whichever the GPU decodes and Windows has a decoder for that
+	// Creates the Media Foundation decoders that decode on the GPU, for H.264,
+	// AV1 and VP9, whichever the GPU decodes and Windows has a decoder for that
 	// uses Direct3D 11. It offers them in the formats WebRTC's software
-	// decoders offer too: H.264 in all its profiles, and AV1 in profile 0.
+	// decoders offer too: H.264 in all its profiles, and AV1 and VP9 in
+	// profile 0.
 	class MFVideoDecoderFactory : public webrtc::VideoDecoderFactory
 	{
 		public:
-			// Returns a factory, or null if the GPU decodes neither codec.
+			// Returns a factory, or null if the GPU decodes none of the codecs.
 			static std::unique_ptr<MFVideoDecoderFactory> Create();
 
 			~MFVideoDecoderFactory() override = default;
@@ -44,11 +45,12 @@ namespace jni
 				const webrtc::SdpVideoFormat & format) override;
 
 		private:
-			MFVideoDecoderFactory(bool h264, bool av1);
+			MFVideoDecoderFactory(bool h264, bool av1, bool vp9);
 
 		private:
 			const bool h264;
 			const bool av1;
+			const bool vp9;
 	};
 }
 

webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoder.cpp

@@ -56,7 +56,14 @@ namespace jni
 
 		const GUID & InputFormat(webrtc::VideoCodecType codec)
 		{
-			return codec == webrtc::kVideoCodecAV1 ? MFVideoFormat_AV1 : MFVideoFormat_H264;
+			switch (codec) {
+				case webrtc::kVideoCodecAV1:
+					return MFVideoFormat_AV1;
+				case webrtc::kVideoCodecVP9:
+					return MFVideoFormat_VP90;
+				default:
+					return MFVideoFormat_H264;
+			}
 		}
 	}
 
@@ -174,6 +181,16 @@ namespace jni
 
 		if (SUCCEEDED(hr)) {
 			hr = SetOutputType();
+
+			// A VP9 stream states its size in the key frames only, and its
+			// decoder may offer no output type before it has seen one. It then
+			// asks for the type again with MF_E_TRANSFORM_STREAM_CHANGE, which
+			// DrainOutput answers, and a decoder that cannot give one at all
+			// fails there, and the software decoder takes over.
+			if (FAILED(hr) && codec == webrtc::kVideoCodecVP9 && !resolution.Valid()) {
+				RTC_LOG(LS_INFO) << implementationName << " offers no output type yet, hr=" << hr;
+				hr = S_OK;
+			}
 		}
 		if (SUCCEEDED(hr)) {
 			hr = transform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
@@ -241,6 +258,15 @@ namespace jni
 			return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
 		}
 
+		// The layers of a VP9 frame with spatial layers reach a decoder back to
+		// back, without the superframe index that tells where one ends. libvpx
+		// takes them that way; a hardware decoder is not known to.
+		if (codec == webrtc::kVideoCodecVP9
+			&& (image.SpatialIndex().value_or(0) > 0 || image.SpatialLayerFrameSize(1).has_value()))
+		{
+			return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+		}
+
 		ComPtr<IMFMediaBuffer> buffer;
 		HRESULT hr = MFCreateMemoryBuffer(static_cast<DWORD>(image.size()), &buffer);
 

webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp

@@ -58,6 +58,7 @@ namespace jni
 	{
 		bool h264 = false;
 		bool av1 = false;
+		bool vp9 = false;
 
 		try {
 			ComInitializer comInitializer;
@@ -86,23 +87,27 @@ namespace jni
 				&& HasDirect3DDecoder(MFVideoFormat_H264, manager.Get());
 			av1 = SupportsDecoderProfile(device.Get(), D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0)
 				&& HasDirect3DDecoder(MFVideoFormat_AV1, manager.Get());
+			vp9 = SupportsDecoderProfile(device.Get(), D3D11_DECODER_PROFILE_VP9_VLD_PROFILE0)
+				&& HasDirect3DDecoder(MFVideoFormat_VP90, manager.Get());
 		}
 		catch (...) {
 			return nullptr;
 		}
 
-		RTC_LOG(LS_INFO) << "Media Foundation hardware decoders, H.264: " << h264 << ", AV1: " << av1;
+		RTC_LOG(LS_INFO) << "Media Foundation hardware decoders, H.264: " << h264 << ", AV1: " << av1
+			<< ", VP9: " << vp9;
 
-		if (!h264 && !av1) {
+		if (!h264 && !av1 && !vp9) {
 			return nullptr;
 		}
 
-		return std::unique_ptr<MFVideoDecoderFactory>(new MFVideoDecoderFactory(h264, av1));
+		return std::unique_ptr<MFVideoDecoderFactory>(new MFVideoDecoderFactory(h264, av1, vp9));
 	}
 
-	MFVideoDecoderFactory::MFVideoDecoderFactory(bool h264, bool av1) :
+	MFVideoDecoderFactory::MFVideoDecoderFactory(bool h264, bool av1, bool vp9) :
 		h264(h264),
-		av1(av1)
+		av1(av1),
+		vp9(vp9)
 	{
 	}
 
@@ -116,6 +121,9 @@ namespace jni
 		if (av1) {
 			formats.push_back(webrtc::SdpVideoFormat::AV1Profile0());
 		}
+		if (vp9) {
+			formats.push_back(webrtc::SdpVideoFormat::VP9Profile0());
+		}
 
 		return formats;
 	}

webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java

@@ -34,8 +34,11 @@
  *     .build();
  * }</pre>
  * <p>
- * On Windows, H.264 and AV1 are decoded on the GPU through Direct3D 11 by the
- * Media Foundation decoders of Windows, where the GPU decodes the codec.
+ * On Windows, H.264, AV1 and VP9 (profile 0) are decoded on the GPU through
+ * Direct3D 11 by the Media Foundation decoders of Windows, where the GPU
+ * decodes the codec and Windows has a decoder for it; VP9 needs the VP9 Video
+ * Extensions of the Microsoft Store. VP9 streams with spatial layers go to
+ * libvpx.
  * Decoded frames are copied back to system memory, which WebRTC's frames are
  * in, so hardware decoding saves CPU mostly at high resolutions. A hardware
  * decoder that fails to start, or fails while decoding, is replaced by the

webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java

@@ -46,7 +46,9 @@
  * A machine without a hardware decoder skips the tests that need one, as CI
  * runners do. Set the system property {@code webrtc.test.hardwareDecoder} to
  * {@code true} on a machine that has one, to make those tests fail instead,
- * and {@code webrtc.test.hardwareAv1Decoder} for a GPU that decodes AV1.
+ * and {@code webrtc.test.hardwareAv1Decoder} for a GPU that decodes AV1, and
+ * {@code webrtc.test.hardwareVp9Decoder} for one that decodes VP9 on Windows.
+ * On macOS the VP9 tests follow {@code webrtc.test.hardwareDecoder}.
  */
 @Execution(ExecutionMode.SAME_THREAD)
 class HardwareVideoDecoderIntegrationTest extends TestBase {
@@ -57,6 +59,8 @@ class HardwareVideoDecoderIntegrationTest extends TestBase {
 
 	private static final boolean HARDWARE_AV1_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareAv1Decoder");
 
+	private static final boolean HARDWARE_VP9_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareVp9Decoder");
+
 	private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT);
 
 	private static final Predicate<RTCRtpCodecCapability> H264 = codec ->
@@ -138,14 +142,10 @@ void macDecodesH264WithVideoToolbox() throws Exception {
 	}
 
 	@Test
-	void macDecodesVp9WithVideoToolbox() throws Exception {
-		assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only");
-
-		PeerConnectionFactory hardware = PeerConnectionFactory.builder()
-				.setAudioDeviceModule(audioDevModule)
-				.setVideoDecoderFactory(new HardwareVideoDecoderFactory())
-				.build();
+	void hardwareDecodesVp9() throws Exception {
+		assumeVp9Platform();
 
+		PeerConnectionFactory hardware = hardwareFactory();
 		String implementation;
 
 		try {
@@ -155,29 +155,22 @@ void macDecodesVp9WithVideoToolbox() throws Exception {
 			hardware.dispose();
 		}
 
-		boolean hardwareUsed = implementation.contains("VideoToolbox");
-
-		if (HARDWARE_REQUIRED) {
-			assertTrue(hardwareUsed, implementation);
-		}
-		else {
-			assumeTrue(hardwareUsed, "no hardware decoder: " + implementation);
-		}
+		expectVp9Hardware(implementation);
 	}
 
 	@Test
-	void macDecodesVp9InSoftwareByDefault() throws Exception {
-		assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only");
+	void defaultDecodesVp9InSoftware() throws Exception {
+		assumeVp9Platform();
 
 		// Hardware decoding is opt-in; the shared factory decodes VP9 with libvpx.
 		String implementation = decoderImplementation(factory, VP9);
 
-		assertFalse(implementation.contains("VideoToolbox"), implementation);
+		assertFalse(isHardware(implementation), implementation);
 	}
 
 	@Test
-	void macFollowsResolutionChange() throws Exception {
-		assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only");
+	void hardwareFollowsResolutionChange() throws Exception {
+		assumeVp9Platform();
 
 		PeerConnectionFactory hardware = PeerConnectionFactory.builder()
 				.setAudioDeviceModule(audioDevModule)
@@ -213,7 +206,7 @@ else if (width == 160 && frame.buffer.getHeight() == 120) {
 
 			String implementation = decoderImplementationOf(call);
 
-			assumeTrue(implementation.contains("VideoToolbox"), "no hardware decoder: " + implementation);
+			expectVp9Hardware(implementation);
 
 			// The sender restarts at half the size, with a key frame; the
 			// decoder has to start a session for it.
@@ -229,7 +222,7 @@ else if (width == 160 && frame.buffer.getHeight() == 120) {
 			assertTrue(half.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "no frames at the new size");
 
 			// Still the hardware decoder, not a fallback.
-			assertTrue(decoderImplementationOf(call).contains("VideoToolbox"));
+			assertTrue(isHardware(decoderImplementationOf(call)));
 
 			track.removeSink(sink);
 			receiver.dispose();
@@ -240,12 +233,12 @@ else if (width == 160 && frame.buffer.getHeight() == 120) {
 	}
 
 	@Test
-	void macVp9NeedsNoKeyFrames() throws Exception {
-		assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only");
+	void vp9NeedsNoKeyFrames() throws Exception {
+		assumeVp9Platform();
 
 		CallResult result = receiveVideo(hardwareFactory(), VP9, 320, 240, null, 60);
 
-		assumeTrue(result.decoder.contains("VideoToolbox"), "no hardware decoder: " + result.decoder);
+		expectVp9Hardware(result.decoder);
 
 		// An inter frame that fails to decode makes the receiver ask for a key
 		// frame, and every key frame is then the only frame that decodes. A
@@ -255,30 +248,31 @@ void macVp9NeedsNoKeyFrames() throws Exception {
 	}
 
 	@Test
-	void macVp9TemporalLayers() throws Exception {
-		assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only");
+	void vp9TemporalLayers() throws Exception {
+		assumeVp9Platform();
 
-		// Temporal layers are one spatial layer, which VideoToolbox decodes.
+		// Temporal layers are one spatial layer, which the hardware decodes.
 		CallResult result = receiveVideo(hardwareFactory(), VP9, 320, 240, "L1T3", 60);
 
-		assumeTrue(result.decoder.contains("VideoToolbox"), "no hardware decoder: " + result.decoder);
+		expectVp9Hardware(result.decoder);
 
 		assertTrue(result.pliCount <= 1, "key frames requested: " + result.pliCount);
 	}
 
 	@Test
-	void macVp9SpatialLayers() throws Exception {
-		assumeTrue(OS.contains("mac"), "VideoToolbox is available on macOS only");
+	void vp9SpatialLayers() throws Exception {
+		assumeVp9Platform();
 
 		// WebRTC encodes spatial layers only above a size, and drops them
 		// below it.
 		CallResult result = receiveVideo(hardwareFactory(), VP9, 640, 480, "L2T2", 60);
 
 		assumeTrue(result.scalability.startsWith("L2"), "no spatial layers were encoded: " + result.scalability);
 
-		// The frames arrive, from libvpx: VideoToolbox does not decode a
-		// frame whose layers come without a superframe index.
-		assertFalse(result.decoder.contains("VideoToolbox"), result.decoder);
+		// The frames arrive, from libvpx: the layers of a frame come without a
+		// superframe index, which VideoToolbox does not decode, and which a
+		// Media Foundation decoder is not known to.
+		assertFalse(isHardware(result.decoder), result.decoder);
 	}
 
 	private PeerConnectionFactory hardwareFactory() {
@@ -288,6 +282,33 @@ private PeerConnectionFactory hardwareFactory() {
 				.build();
 	}
 
+	/**
+	 * Skips the test where VP9 is not decoded in hardware: macOS and Windows.
+	 */
+	private static void assumeVp9Platform() {
+		assumeTrue(OS.contains("mac") || OS.contains("win"),
+				"VP9 is decoded in hardware on macOS and Windows only");
+	}
+
+	private static boolean isHardware(String implementation) {
+		return implementation.contains("VideoToolbox") || implementation.contains("MediaFoundation");
+	}
+
+	/**
+	 * Where a hardware decoder is required, the test fails without one;
+	 * elsewhere it is skipped.
+	 */
+	private static void expectVp9Hardware(String implementation) {
+		boolean required = OS.contains("mac") ? HARDWARE_REQUIRED : HARDWARE_VP9_REQUIRED;
+
+		if (required) {
+			assertTrue(isHardware(implementation), implementation);
+		}
+		else {
+			assumeTrue(isHardware(implementation), "no hardware decoder: " + implementation);
+		}
+	}
+
 	/**
 	 * What a call that received video tells about its decoder.
 	 */