feat: decode H.265 and AV1 with NVDEC on Linux

docs/guide/advanced/video-codecs.md

@@ -68,7 +68,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
 | Platform | Hardware decoding |
 |---|---|
 | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) |
-| Linux | H.264 and VP9 (profile 0) on NVIDIA GPUs, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` |
+| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` |
 | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer |
 
 On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software.

webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h

@@ -58,7 +58,8 @@ namespace jni
 	{
 		public:
 			// Returns the library, or null if there is no NVIDIA driver, no
-			// CUDA device, or a device that decodes neither H.264 nor VP9.
+			// CUDA device, or a device that decodes none of H.264, H.265, VP9
+			// and AV1.
 			static NvdecLibrary * Get();
 
 			NvdecLibrary(const NvdecLibrary &) = delete;
@@ -69,8 +70,12 @@ namespace jni
 			// The name of the device, e.g. "NVIDIA GeForce RTX 4070".
 			const std::string & DeviceName() const;
 
+			// Whether the device decodes the codec at all, 8 bit 4:2:0. AV1
+			// only newer GPUs do (since Ampere).
 			bool SupportsH264() const;
+			bool SupportsH265() const;
 			bool SupportsVp9() const;
+			bool SupportsAv1() const;
 
 			// Whether the device decodes the codec, 8 bit 4:2:0, at this
 			// coded size. Needs the context current.
@@ -100,7 +105,9 @@ namespace jni
 			CUdevice device = 0;
 			std::string deviceName;
 			bool h264 = false;
+			bool h265 = false;
 			bool vp9 = false;
+			bool av1 = false;
 	};
 
 	// Makes a CUDA context current on the calling thread for as long as it

webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h

@@ -31,7 +31,8 @@
 
 namespace jni
 {
-	// Decodes H.264 or VP9 profile 0 on the NVDEC engine of an NVIDIA GPU.
+	// Decodes H.264, H.265 Main, VP9 profile 0 or AV1 profile 0 on the NVDEC
+	// engine of an NVIDIA GPU.
 	//
 	// NVDEC's own parser takes the encoded image, finds the pictures in it and
 	// calls back in three steps: a sequence, which creates the decoder, a
@@ -44,11 +45,12 @@ namespace jni
 	// Whatever NVDEC cannot decode, such as another profile or a size the GPU
 	// does not take, and whatever fails, returns
 	// WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so that the software decoder takes
-	// over.
+	// over. H.265 has none, so for it that is an error, and WebRTC asks the
+	// sender for a key frame.
 	class NvdecVideoDecoder : public webrtc::VideoDecoder
 	{
 		public:
-			// The codec is H.264 or VP9.
+			// The codec is H.264, H.265, VP9 or AV1.
 			NvdecVideoDecoder(NvdecLibrary & library, webrtc::VideoCodecType codec);
 			~NvdecVideoDecoder() override;
 
@@ -113,6 +115,9 @@ namespace jni
 			// The decoded picture in system memory, NV12.
 			std::vector<uint8_t> nv12;
 
+			// An AV1 frame with the temporal delimiter put in front of it.
+			std::vector<uint8_t> av1Unit;
+
 			int64_t counter;
 			std::map<int64_t, PendingFrame> pendingFrames;
 	};

webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h

@@ -29,10 +29,11 @@
 
 namespace jni
 {
-	// Creates the NVDEC decoders that decode on an NVIDIA GPU, for H.264 and
-	// VP9, whichever the GPU decodes. It offers them in the formats WebRTC's
-	// software decoders offer too: H.264 in all its profiles, and VP9 in
-	// profile 0.
+	// Creates the NVDEC decoders that decode on an NVIDIA GPU, for H.264,
+	// H.265, VP9 and AV1, whichever the GPU decodes. It offers them in the
+	// formats WebRTC's software decoders offer too: H.264 in all its profiles,
+	// and VP9 and AV1 in profile 0. H.265, which WebRTC has no decoder of its
+	// own for, is offered in Main profile.
 	class NvdecVideoDecoderFactory : public webrtc::VideoDecoderFactory
 	{
 		public:

webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp

@@ -49,8 +49,9 @@ namespace jni
 	{
 		std::vector<std::unique_ptr<webrtc::VideoDecoderFactory>> factories;
 
-		// NVDEC on NVIDIA GPUs; VA-API for the GPUs of Intel and AMD, only if
-		// asked for, since it has not been tested on such a GPU yet.
+		// NVDEC on NVIDIA GPUs, for H.264, H.265, VP9 and AV1; VA-API for the
+		// GPUs of Intel and AMD, only if asked for, since it has not been
+		// tested on such a GPU yet.
 		if (auto nvdec = NvdecVideoDecoderFactory::Create()) {
 			factories.push_back(std::move(nvdec));
 		}

webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp

@@ -103,19 +103,22 @@ namespace jni
 
 		if (PushContext(context)) {
 			h264 = QueryCodec(cudaVideoCodec_H264);
+			h265 = QueryCodec(cudaVideoCodec_HEVC);
 			vp9 = QueryCodec(cudaVideoCodec_VP9);
+			av1 = QueryCodec(cudaVideoCodec_AV1);
 
 			PopContext();
 		}
 
 		ReleaseContext();
 
-		if (!h264 && !vp9) {
-			RTC_LOG(LS_INFO) << "NVDEC: " << deviceName << " decodes neither H.264 nor VP9";
+		if (!h264 && !h265 && !vp9 && !av1) {
+			RTC_LOG(LS_INFO) << "NVDEC: " << deviceName << " decodes none of H.264, H.265, VP9 and AV1";
 			return false;
 		}
 
-		RTC_LOG(LS_INFO) << "NVDEC available on " << deviceName << ", H.264: " << h264 << ", VP9: " << vp9;
+		RTC_LOG(LS_INFO) << "NVDEC available on " << deviceName << ", H.264: " << h264 << ", H.265: " << h265
+			<< ", VP9: " << vp9 << ", AV1: " << av1;
 
 		return true;
 	}
@@ -162,11 +165,21 @@ namespace jni
 		return h264;
 	}
 
+	bool NvdecLibrary::SupportsH265() const
+	{
+		return h265;
+	}
+
 	bool NvdecLibrary::SupportsVp9() const
 	{
 		return vp9;
 	}
 
+	bool NvdecLibrary::SupportsAv1() const
+	{
+		return av1;
+	}
+
 	bool NvdecLibrary::RetainContext(CUcontext * context) const
 	{
 		return api.cuDevicePrimaryCtxRetain(context, device) == CUDA_SUCCESS;

webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp

@@ -31,7 +31,26 @@ namespace jni
 	{
 		cudaVideoCodec CudaCodecOf(webrtc::VideoCodecType codec)
 		{
-			return codec == webrtc::kVideoCodecVP9 ? cudaVideoCodec_VP9 : cudaVideoCodec_H264;
+			switch (codec) {
+				case webrtc::kVideoCodecVP9:
+					return cudaVideoCodec_VP9;
+				case webrtc::kVideoCodecH265:
+					return cudaVideoCodec_HEVC;
+				case webrtc::kVideoCodecAV1:
+					return cudaVideoCodec_AV1;
+				default:
+					return cudaVideoCodec_H264;
+			}
+		}
+
+		// An AV1 temporal delimiter OBU: type 2, with a size field of 0
+		// (section 5.6 of the AV1 specification).
+		constexpr uint8_t kAv1TemporalDelimiter[] = { 0x12, 0x00 };
+
+		bool StartsWithTemporalDelimiter(const uint8_t * data, size_t size)
+		{
+			// The OBU type is in bits 3 to 6 of the header.
+			return size > 0 && ((data[0] >> 3) & 0x0F) == 2;
 		}
 
 		// How many entries of a frame that never came out are kept.
@@ -131,6 +150,11 @@ namespace jni
 		{
 			return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
 		}
+		// The frames of an AV1 spatial layer above the first likewise go to
+		// dav1d, which decodes them as they come.
+		if (codec == webrtc::kVideoCodecAV1 && image.SpatialIndex().value_or(0) > 0) {
+			return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+		}
 
 		// The parser needs the parameters of the stream, which a key frame
 		// carries. Until one has come, WebRTC is asked for it.
@@ -164,6 +188,16 @@ namespace jni
 		packet.payload_size = image.size();
 		packet.timestamp = timestamp;
 
+		// NVDEC's AV1 parser takes temporal units, which begin with a
+		// temporal delimiter; WebRTC's frames come without one.
+		if (codec == webrtc::kVideoCodecAV1 && !StartsWithTemporalDelimiter(image.data(), image.size())) {
+			av1Unit.assign(std::begin(kAv1TemporalDelimiter), std::end(kAv1TemporalDelimiter));
+			av1Unit.insert(av1Unit.end(), image.data(), image.data() + image.size());
+
+			packet.payload = av1Unit.data();
+			packet.payload_size = av1Unit.size();
+		}
+
 		const CUresult result = library.Api().cuvidParseVideoData(parser, &packet);
 
 		if (result != CUDA_SUCCESS || failed) {

webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp

@@ -15,6 +15,7 @@
  */
 
 #include "media/video/codec/nvdec/NvdecVideoDecoderFactory.h"
+#include "media/video/codec/H265Format.h"
 #include "media/video/codec/nvdec/NvdecVideoDecoder.h"
 
 #include "api/video_codecs/video_codec.h"
@@ -45,9 +46,15 @@ namespace jni
 		if (library.SupportsH264()) {
 			formats = webrtc::SupportedH264DecoderCodecs();
 		}
+		if (library.SupportsH265()) {
+			formats.push_back(CreateH265MainFormat());
+		}
 		if (library.SupportsVp9()) {
 			formats.push_back(webrtc::SdpVideoFormat::VP9Profile0());
 		}
+		if (library.SupportsAv1()) {
+			formats.push_back(webrtc::SdpVideoFormat::AV1Profile0());
+		}
 
 		return formats;
 	}

webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java

@@ -51,12 +51,14 @@
  * to libvpx and dav1d. Where there is no hardware decoder, this factory
  * decodes like a {@link DefaultVideoDecoderFactory}.
  * <p>
- * On Linux, H.264 and VP9 (profile 0) are decoded on NVIDIA GPUs with NVDEC,
- * loaded from the NVIDIA driver when the factory is made; a machine without
- * the driver is not affected. Streams with spatial layers go to libvpx.
- * VP9 (profile 0) is decoded with VA-API on the GPUs of Intel and AMD only
- * with the system property {@code webrtc.hardware.vaapiExperimental} set to
- * {@code true}: the decoder has not been tested on such a GPU yet.
+ * On Linux, H.264, H.265 (Main profile), VP9 (profile 0) and AV1 (profile 0)
+ * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1
+ * since Ampere), loaded from the NVIDIA driver when the factory is made; a
+ * machine without the driver is not affected. Streams with spatial layers go
+ * to libvpx and dav1d. VP9 (profile 0) is decoded with VA-API on the GPUs of
+ * Intel and AMD only with the system property
+ * {@code webrtc.hardware.vaapiExperimental} set to {@code true}: the decoder
+ * has not been tested on such a GPU yet.
  * <p>
  * The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in
  * the same order. H.265 comes after them, only where the hardware decodes it: