From 08a1215ab99d259249ef32e2fbedc608e867fd73 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 11 Oct 2026 22:38:23 +0200 Subject: [PATCH] feat: decode H.265 and AV1 with NVDEC on Linux The NVDEC decoders decoded H.264 and VP9. They now decode H.265 (Main profile) and AV1 (profile 0) as well, where the GPU does (AV1 since Ampere), asking the driver which codecs it decodes. NVDEC's parser takes the bitstream, so a codec is mostly a matter of telling it which. NVDEC's AV1 parser takes temporal units, which begin with a temporal delimiter, and WebRTC's AV1 frames come without one, so one is put in front. Frames of AV1 spatial layers above the first go to dav1d, as those of VP9 go to libvpx. H.265 has no software decoder to fall back to: what NVDEC cannot decode is an error, and WebRTC asks for a key frame. With NVENC's H.265, NVIDIA GPUs on Linux now carry H.265 calls both ways. --- docs/guide/advanced/video-codecs.md | 2 +- .../media/video/codec/nvdec/NvdecLibrary.h | 9 ++++- .../video/codec/nvdec/NvdecVideoDecoder.h | 11 ++++-- .../codec/nvdec/NvdecVideoDecoderFactory.h | 9 ++--- .../LinuxHardwareVideoCodecFactories.cpp | 5 +-- .../media/video/codec/nvdec/NvdecLibrary.cpp | 19 ++++++++-- .../video/codec/nvdec/NvdecVideoDecoder.cpp | 36 ++++++++++++++++++- .../codec/nvdec/NvdecVideoDecoderFactory.cpp | 7 ++++ .../codec/HardwareVideoDecoderFactory.java | 14 ++++---- 9 files changed, 91 insertions(+), 21 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index d4f2c0d5..45552730 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -68,7 +68,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264 and VP9 (profile 0) on NVIDIA GPUs, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` | +| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h index d91f34cf..e5e78e90 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h @@ -58,7 +58,8 @@ namespace jni { public: // Returns the library, or null if there is no NVIDIA driver, no - // CUDA device, or a device that decodes neither H.264 nor VP9. + // CUDA device, or a device that decodes none of H.264, H.265, VP9 + // and AV1. static NvdecLibrary * Get(); NvdecLibrary(const NvdecLibrary &) = delete; @@ -69,8 +70,12 @@ namespace jni // The name of the device, e.g. "NVIDIA GeForce RTX 4070". const std::string & DeviceName() const; + // Whether the device decodes the codec at all, 8 bit 4:2:0. AV1 + // only newer GPUs do (since Ampere). bool SupportsH264() const; + bool SupportsH265() const; bool SupportsVp9() const; + bool SupportsAv1() const; // Whether the device decodes the codec, 8 bit 4:2:0, at this // coded size. Needs the context current. @@ -100,7 +105,9 @@ namespace jni CUdevice device = 0; std::string deviceName; bool h264 = false; + bool h265 = false; bool vp9 = false; + bool av1 = false; }; // Makes a CUDA context current on the calling thread for as long as it diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h index f7fc6fae..860f73f6 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h @@ -31,7 +31,8 @@ namespace jni { - // Decodes H.264 or VP9 profile 0 on the NVDEC engine of an NVIDIA GPU. + // Decodes H.264, H.265 Main, VP9 profile 0 or AV1 profile 0 on the NVDEC + // engine of an NVIDIA GPU. // // NVDEC's own parser takes the encoded image, finds the pictures in it and // calls back in three steps: a sequence, which creates the decoder, a @@ -44,11 +45,12 @@ namespace jni // Whatever NVDEC cannot decode, such as another profile or a size the GPU // does not take, and whatever fails, returns // WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so that the software decoder takes - // over. + // over. H.265 has none, so for it that is an error, and WebRTC asks the + // sender for a key frame. class NvdecVideoDecoder : public webrtc::VideoDecoder { public: - // The codec is H.264 or VP9. + // The codec is H.264, H.265, VP9 or AV1. NvdecVideoDecoder(NvdecLibrary & library, webrtc::VideoCodecType codec); ~NvdecVideoDecoder() override; @@ -113,6 +115,9 @@ namespace jni // The decoded picture in system memory, NV12. std::vector nv12; + // An AV1 frame with the temporal delimiter put in front of it. + std::vector av1Unit; + int64_t counter; std::map pendingFrames; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h index 92a543ac..ec52b569 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h @@ -29,10 +29,11 @@ namespace jni { - // Creates the NVDEC decoders that decode on an NVIDIA GPU, for H.264 and - // VP9, whichever the GPU decodes. It offers them in the formats WebRTC's - // software decoders offer too: H.264 in all its profiles, and VP9 in - // profile 0. + // Creates the NVDEC decoders that decode on an NVIDIA GPU, for H.264, + // H.265, VP9 and AV1, whichever the GPU decodes. It offers them in the + // formats WebRTC's software decoders offer too: H.264 in all its profiles, + // and VP9 and AV1 in profile 0. H.265, which WebRTC has no decoder of its + // own for, is offered in Main profile. class NvdecVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index 7f144f20..ff33028c 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -49,8 +49,9 @@ namespace jni { std::vector> factories; - // NVDEC on NVIDIA GPUs; VA-API for the GPUs of Intel and AMD, only if - // asked for, since it has not been tested on such a GPU yet. + // NVDEC on NVIDIA GPUs, for H.264, H.265, VP9 and AV1; VA-API for the + // GPUs of Intel and AMD, only if asked for, since it has not been + // tested on such a GPU yet. if (auto nvdec = NvdecVideoDecoderFactory::Create()) { factories.push_back(std::move(nvdec)); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp index 5405a1ed..3bc7deb5 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp @@ -103,19 +103,22 @@ namespace jni if (PushContext(context)) { h264 = QueryCodec(cudaVideoCodec_H264); + h265 = QueryCodec(cudaVideoCodec_HEVC); vp9 = QueryCodec(cudaVideoCodec_VP9); + av1 = QueryCodec(cudaVideoCodec_AV1); PopContext(); } ReleaseContext(); - if (!h264 && !vp9) { - RTC_LOG(LS_INFO) << "NVDEC: " << deviceName << " decodes neither H.264 nor VP9"; + if (!h264 && !h265 && !vp9 && !av1) { + RTC_LOG(LS_INFO) << "NVDEC: " << deviceName << " decodes none of H.264, H.265, VP9 and AV1"; return false; } - RTC_LOG(LS_INFO) << "NVDEC available on " << deviceName << ", H.264: " << h264 << ", VP9: " << vp9; + RTC_LOG(LS_INFO) << "NVDEC available on " << deviceName << ", H.264: " << h264 << ", H.265: " << h265 + << ", VP9: " << vp9 << ", AV1: " << av1; return true; } @@ -162,11 +165,21 @@ namespace jni return h264; } + bool NvdecLibrary::SupportsH265() const + { + return h265; + } + bool NvdecLibrary::SupportsVp9() const { return vp9; } + bool NvdecLibrary::SupportsAv1() const + { + return av1; + } + bool NvdecLibrary::RetainContext(CUcontext * context) const { return api.cuDevicePrimaryCtxRetain(context, device) == CUDA_SUCCESS; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp index cf6d7dfd..ccfee9dd 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp @@ -31,7 +31,26 @@ namespace jni { cudaVideoCodec CudaCodecOf(webrtc::VideoCodecType codec) { - return codec == webrtc::kVideoCodecVP9 ? cudaVideoCodec_VP9 : cudaVideoCodec_H264; + switch (codec) { + case webrtc::kVideoCodecVP9: + return cudaVideoCodec_VP9; + case webrtc::kVideoCodecH265: + return cudaVideoCodec_HEVC; + case webrtc::kVideoCodecAV1: + return cudaVideoCodec_AV1; + default: + return cudaVideoCodec_H264; + } + } + + // An AV1 temporal delimiter OBU: type 2, with a size field of 0 + // (section 5.6 of the AV1 specification). + constexpr uint8_t kAv1TemporalDelimiter[] = { 0x12, 0x00 }; + + bool StartsWithTemporalDelimiter(const uint8_t * data, size_t size) + { + // The OBU type is in bits 3 to 6 of the header. + return size > 0 && ((data[0] >> 3) & 0x0F) == 2; } // How many entries of a frame that never came out are kept. @@ -131,6 +150,11 @@ namespace jni { return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } + // The frames of an AV1 spatial layer above the first likewise go to + // dav1d, which decodes them as they come. + if (codec == webrtc::kVideoCodecAV1 && image.SpatialIndex().value_or(0) > 0) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } // The parser needs the parameters of the stream, which a key frame // carries. Until one has come, WebRTC is asked for it. @@ -164,6 +188,16 @@ namespace jni packet.payload_size = image.size(); packet.timestamp = timestamp; + // NVDEC's AV1 parser takes temporal units, which begin with a + // temporal delimiter; WebRTC's frames come without one. + if (codec == webrtc::kVideoCodecAV1 && !StartsWithTemporalDelimiter(image.data(), image.size())) { + av1Unit.assign(std::begin(kAv1TemporalDelimiter), std::end(kAv1TemporalDelimiter)); + av1Unit.insert(av1Unit.end(), image.data(), image.data() + image.size()); + + packet.payload = av1Unit.data(); + packet.payload_size = av1Unit.size(); + } + const CUresult result = library.Api().cuvidParseVideoData(parser, &packet); if (result != CUDA_SUCCESS || failed) { diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp index 1be1d4ad..3db8133e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp @@ -15,6 +15,7 @@ */ #include "media/video/codec/nvdec/NvdecVideoDecoderFactory.h" +#include "media/video/codec/H265Format.h" #include "media/video/codec/nvdec/NvdecVideoDecoder.h" #include "api/video_codecs/video_codec.h" @@ -45,9 +46,15 @@ namespace jni if (library.SupportsH264()) { formats = webrtc::SupportedH264DecoderCodecs(); } + if (library.SupportsH265()) { + formats.push_back(CreateH265MainFormat()); + } if (library.SupportsVp9()) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); } + if (library.SupportsAv1()) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } return formats; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 4165e1cc..3321751d 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -51,12 +51,14 @@ * to libvpx and dav1d. Where there is no hardware decoder, this factory * decodes like a {@link DefaultVideoDecoderFactory}. *

- * On Linux, H.264 and VP9 (profile 0) are decoded on NVIDIA GPUs with NVDEC, - * loaded from the NVIDIA driver when the factory is made; a machine without - * the driver is not affected. Streams with spatial layers go to libvpx. - * VP9 (profile 0) is decoded with VA-API on the GPUs of Intel and AMD only - * with the system property {@code webrtc.hardware.vaapiExperimental} set to - * {@code true}: the decoder has not been tested on such a GPU yet. + * On Linux, H.264, H.265 (Main profile), VP9 (profile 0) and AV1 (profile 0) + * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1 + * since Ampere), loaded from the NVIDIA driver when the factory is made; a + * machine without the driver is not affected. Streams with spatial layers go + * to libvpx and dav1d. VP9 (profile 0) is decoded with VA-API on the GPUs of + * Intel and AMD only with the system property + * {@code webrtc.hardware.vaapiExperimental} set to {@code true}: the decoder + * has not been tested on such a GPU yet. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. H.265 comes after them, only where the hardware decodes it: