diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index d4f2c0d5..45552730 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -68,7 +68,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264 and VP9 (profile 0) on NVIDIA GPUs, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` | +| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h index d91f34cf..e5e78e90 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecLibrary.h @@ -58,7 +58,8 @@ namespace jni { public: // Returns the library, or null if there is no NVIDIA driver, no - // CUDA device, or a device that decodes neither H.264 nor VP9. + // CUDA device, or a device that decodes none of H.264, H.265, VP9 + // and AV1. static NvdecLibrary * Get(); NvdecLibrary(const NvdecLibrary &) = delete; @@ -69,8 +70,12 @@ namespace jni // The name of the device, e.g. "NVIDIA GeForce RTX 4070". const std::string & DeviceName() const; + // Whether the device decodes the codec at all, 8 bit 4:2:0. AV1 + // only newer GPUs do (since Ampere). bool SupportsH264() const; + bool SupportsH265() const; bool SupportsVp9() const; + bool SupportsAv1() const; // Whether the device decodes the codec, 8 bit 4:2:0, at this // coded size. Needs the context current. @@ -100,7 +105,9 @@ namespace jni CUdevice device = 0; std::string deviceName; bool h264 = false; + bool h265 = false; bool vp9 = false; + bool av1 = false; }; // Makes a CUDA context current on the calling thread for as long as it diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h index f7fc6fae..860f73f6 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoder.h @@ -31,7 +31,8 @@ namespace jni { - // Decodes H.264 or VP9 profile 0 on the NVDEC engine of an NVIDIA GPU. + // Decodes H.264, H.265 Main, VP9 profile 0 or AV1 profile 0 on the NVDEC + // engine of an NVIDIA GPU. // // NVDEC's own parser takes the encoded image, finds the pictures in it and // calls back in three steps: a sequence, which creates the decoder, a @@ -44,11 +45,12 @@ namespace jni // Whatever NVDEC cannot decode, such as another profile or a size the GPU // does not take, and whatever fails, returns // WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so that the software decoder takes - // over. + // over. H.265 has none, so for it that is an error, and WebRTC asks the + // sender for a key frame. class NvdecVideoDecoder : public webrtc::VideoDecoder { public: - // The codec is H.264 or VP9. + // The codec is H.264, H.265, VP9 or AV1. NvdecVideoDecoder(NvdecLibrary & library, webrtc::VideoCodecType codec); ~NvdecVideoDecoder() override; @@ -113,6 +115,9 @@ namespace jni // The decoded picture in system memory, NV12. std::vector nv12; + // An AV1 frame with the temporal delimiter put in front of it. + std::vector av1Unit; + int64_t counter; std::map pendingFrames; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h index 92a543ac..ec52b569 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/nvdec/NvdecVideoDecoderFactory.h @@ -29,10 +29,11 @@ namespace jni { - // Creates the NVDEC decoders that decode on an NVIDIA GPU, for H.264 and - // VP9, whichever the GPU decodes. It offers them in the formats WebRTC's - // software decoders offer too: H.264 in all its profiles, and VP9 in - // profile 0. + // Creates the NVDEC decoders that decode on an NVIDIA GPU, for H.264, + // H.265, VP9 and AV1, whichever the GPU decodes. It offers them in the + // formats WebRTC's software decoders offer too: H.264 in all its profiles, + // and VP9 and AV1 in profile 0. H.265, which WebRTC has no decoder of its + // own for, is offered in Main profile. class NvdecVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index 7f144f20..ff33028c 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -49,8 +49,9 @@ namespace jni { std::vector> factories; - // NVDEC on NVIDIA GPUs; VA-API for the GPUs of Intel and AMD, only if - // asked for, since it has not been tested on such a GPU yet. + // NVDEC on NVIDIA GPUs, for H.264, H.265, VP9 and AV1; VA-API for the + // GPUs of Intel and AMD, only if asked for, since it has not been + // tested on such a GPU yet. if (auto nvdec = NvdecVideoDecoderFactory::Create()) { factories.push_back(std::move(nvdec)); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp index 5405a1ed..3bc7deb5 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecLibrary.cpp @@ -103,19 +103,22 @@ namespace jni if (PushContext(context)) { h264 = QueryCodec(cudaVideoCodec_H264); + h265 = QueryCodec(cudaVideoCodec_HEVC); vp9 = QueryCodec(cudaVideoCodec_VP9); + av1 = QueryCodec(cudaVideoCodec_AV1); PopContext(); } ReleaseContext(); - if (!h264 && !vp9) { - RTC_LOG(LS_INFO) << "NVDEC: " << deviceName << " decodes neither H.264 nor VP9"; + if (!h264 && !h265 && !vp9 && !av1) { + RTC_LOG(LS_INFO) << "NVDEC: " << deviceName << " decodes none of H.264, H.265, VP9 and AV1"; return false; } - RTC_LOG(LS_INFO) << "NVDEC available on " << deviceName << ", H.264: " << h264 << ", VP9: " << vp9; + RTC_LOG(LS_INFO) << "NVDEC available on " << deviceName << ", H.264: " << h264 << ", H.265: " << h265 + << ", VP9: " << vp9 << ", AV1: " << av1; return true; } @@ -162,11 +165,21 @@ namespace jni return h264; } + bool NvdecLibrary::SupportsH265() const + { + return h265; + } + bool NvdecLibrary::SupportsVp9() const { return vp9; } + bool NvdecLibrary::SupportsAv1() const + { + return av1; + } + bool NvdecLibrary::RetainContext(CUcontext * context) const { return api.cuDevicePrimaryCtxRetain(context, device) == CUDA_SUCCESS; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp index cf6d7dfd..ccfee9dd 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoder.cpp @@ -31,7 +31,26 @@ namespace jni { cudaVideoCodec CudaCodecOf(webrtc::VideoCodecType codec) { - return codec == webrtc::kVideoCodecVP9 ? cudaVideoCodec_VP9 : cudaVideoCodec_H264; + switch (codec) { + case webrtc::kVideoCodecVP9: + return cudaVideoCodec_VP9; + case webrtc::kVideoCodecH265: + return cudaVideoCodec_HEVC; + case webrtc::kVideoCodecAV1: + return cudaVideoCodec_AV1; + default: + return cudaVideoCodec_H264; + } + } + + // An AV1 temporal delimiter OBU: type 2, with a size field of 0 + // (section 5.6 of the AV1 specification). + constexpr uint8_t kAv1TemporalDelimiter[] = { 0x12, 0x00 }; + + bool StartsWithTemporalDelimiter(const uint8_t * data, size_t size) + { + // The OBU type is in bits 3 to 6 of the header. + return size > 0 && ((data[0] >> 3) & 0x0F) == 2; } // How many entries of a frame that never came out are kept. @@ -131,6 +150,11 @@ namespace jni { return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } + // The frames of an AV1 spatial layer above the first likewise go to + // dav1d, which decodes them as they come. + if (codec == webrtc::kVideoCodecAV1 && image.SpatialIndex().value_or(0) > 0) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } // The parser needs the parameters of the stream, which a key frame // carries. Until one has come, WebRTC is asked for it. @@ -164,6 +188,16 @@ namespace jni packet.payload_size = image.size(); packet.timestamp = timestamp; + // NVDEC's AV1 parser takes temporal units, which begin with a + // temporal delimiter; WebRTC's frames come without one. + if (codec == webrtc::kVideoCodecAV1 && !StartsWithTemporalDelimiter(image.data(), image.size())) { + av1Unit.assign(std::begin(kAv1TemporalDelimiter), std::end(kAv1TemporalDelimiter)); + av1Unit.insert(av1Unit.end(), image.data(), image.data() + image.size()); + + packet.payload = av1Unit.data(); + packet.payload_size = av1Unit.size(); + } + const CUresult result = library.Api().cuvidParseVideoData(parser, &packet); if (result != CUDA_SUCCESS || failed) { diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp index 1be1d4ad..3db8133e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp @@ -15,6 +15,7 @@ */ #include "media/video/codec/nvdec/NvdecVideoDecoderFactory.h" +#include "media/video/codec/H265Format.h" #include "media/video/codec/nvdec/NvdecVideoDecoder.h" #include "api/video_codecs/video_codec.h" @@ -45,9 +46,15 @@ namespace jni if (library.SupportsH264()) { formats = webrtc::SupportedH264DecoderCodecs(); } + if (library.SupportsH265()) { + formats.push_back(CreateH265MainFormat()); + } if (library.SupportsVp9()) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); } + if (library.SupportsAv1()) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } return formats; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 4165e1cc..3321751d 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -51,12 +51,14 @@ * to libvpx and dav1d. Where there is no hardware decoder, this factory * decodes like a {@link DefaultVideoDecoderFactory}. *

- * On Linux, H.264 and VP9 (profile 0) are decoded on NVIDIA GPUs with NVDEC, - * loaded from the NVIDIA driver when the factory is made; a machine without - * the driver is not affected. Streams with spatial layers go to libvpx. - * VP9 (profile 0) is decoded with VA-API on the GPUs of Intel and AMD only - * with the system property {@code webrtc.hardware.vaapiExperimental} set to - * {@code true}: the decoder has not been tested on such a GPU yet. + * On Linux, H.264, H.265 (Main profile), VP9 (profile 0) and AV1 (profile 0) + * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1 + * since Ampere), loaded from the NVIDIA driver when the factory is made; a + * machine without the driver is not affected. Streams with spatial layers go + * to libvpx and dav1d. VP9 (profile 0) is decoded with VA-API on the GPUs of + * Intel and AMD only with the system property + * {@code webrtc.hardware.vaapiExperimental} set to {@code true}: the decoder + * has not been tested on such a GPU yet. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. H.265 comes after them, only where the hardware decodes it: