diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index e185733e3..d4f2c0d52 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -27,12 +27,12 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | H.264 | H.265 | AV1 | |---|---|---|---| | Windows | NVENC on NVIDIA GPUs, otherwise the Media Foundation encoder of the GPU driver (AMD, Intel) | NVENC on NVIDIA GPUs, otherwise Media Foundation on GPUs that encode H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise Media Foundation on GPUs that encode AV1 (e.g. AMD RDNA3 and newer, Intel Arc) | -| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs | NVENC on NVIDIA RTX 40 and newer | +| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs; VA-API (Intel, AMD) when turned on, see below | NVENC on NVIDIA RTX 40 and newer | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. -VP9 can be encoded in hardware too, on Windows with the Media Foundation VP9 encoder of the GPU driver, which Intel GPUs have. It is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: +VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through Media Foundation on Windows and VA-API on Linux. It is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: ```java System.setProperty("webrtc.hardware.vp9Encoder", "true"); @@ -40,6 +40,14 @@ System.setProperty("webrtc.hardware.vp9Encoder", "true"); Where the GPU has no VP9 encoder, or the stream asks for SVC, VP9 is still encoded with libvpx. +On Linux, H.265 can be encoded with the VA-API encoder of Intel and AMD drivers, also off by default until it has been tested on such GPUs: + +```java +System.setProperty("webrtc.hardware.vaapiExperimental", "true"); +``` + +It is offered only where the driver encodes H.265 Main, and only for sending: Intel and AMD GPUs do not decode H.265 here yet, so the other end has to. + NVENC needs an NVIDIA driver of version 522 or newer on Windows, 520 or newer on Linux. VA-API needs libva 2 and a driver that encodes H.264, such as Intel's `intel-media-va-driver` (iHD) or Mesa's `mesa-va-drivers` for AMD, and access to a render node in `/dev/dri`. All of them are loaded at run time, so nothing needs to be installed on machines without them. The hardware encoders take over H.264 Constrained Baseline and Baseline with packetization mode 1, and AV1 profile 0, formats the software encoders offer too, so encoding these in hardware never changes what is negotiated. H.265 (Main profile) has no software encoder: it is offered after the other codecs, only where the hardware encodes it, so the codec a peer connection prefers stays the same, and a call uses H.265 only if it is preferred (e.g. with `RTCRtpTransceiver.setCodecPreferences()`) and the other end decodes it. On macOS, VideoToolbox is required to encode H.265 in hardware, and never falls back to its software encoder; a Mac without a hardware HEVC encoder does not offer H.265. When a hardware encoder fails to start, for example because the GPU has no encoder sessions left, or fails while encoding, the stream switches to the next encoder in line (e.g. from NVENC to Media Foundation on Windows), and finally to the software encoder, and continues with a key frame. On a machine without a hardware encoder, the factory encodes like `DefaultVideoEncoderFactory`. @@ -60,7 +68,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264 and VP9 (profile 0) on NVIDIA GPUs, through NVDEC; VA-API, for Intel and AMD GPUs, is not there yet | +| Linux | H.264 and VP9 (profile 0) on NVIDIA GPUs, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h index 2b60dcd32..81c151d85 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h @@ -24,10 +24,18 @@ namespace jni // tested on one. constexpr const char * kHardwareVp9EncoderProperty = "webrtc.hardware.vp9Encoder"; + // The Java system property that turns on the VA-API codecs beyond H.264 + // encoding, which are off by default: they have not been tested on an + // Intel or AMD GPU under Linux yet. + constexpr const char * kExperimentalVaapiProperty = "webrtc.hardware.vaapiExperimental"; + // Whether hardware VP9 encoding is turned on: the Java system property // above is "true", as Boolean.getBoolean has it. Callable from any thread; // false where Java cannot be asked, for one with an exception pending. bool IsHardwareVp9EncoderEnabled(); + + // Whether the experimental VA-API codecs are turned on, the same way. + bool IsExperimentalVaapiEnabled(); } #endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h new file mode 100644 index 000000000..32cd6408b --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_ENCODE_SESSION_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_ENCODE_SESSION_H_ + +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/video_frame.h" + +#include + +#include +#include +#include + +namespace jni +{ + // What every VA-API encoder of a single codec needs: a configuration and + // a context with an input surface and two reconstructed ones (the frame + // being encoded and the one it refers to), the output buffer, the + // parameter buffers of a frame, uploading a frame as NV12, encoding it + // and reading what came out. The codec's encoder fills in the parameters. + // + // Used by the H.265 and VP9 encoders; the H.264 one keeps its own, older + // copy of the same. + class VaapiEncodeSession + { + public: + // The name says whose operations failed, in the log. + VaapiEncodeSession(VaapiLibrary & library, std::string name); + ~VaapiEncodeSession(); + + VaapiEncodeSession(const VaapiEncodeSession &) = delete; + VaapiEncodeSession & operator=(const VaapiEncodeSession &) = delete; + + // Creates the session for 4:2:0 at constant bitrate, with surfaces + // of the given size, which is that of the frame aligned as the + // codec needs. + bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t surfaceWidth, uint32_t surfaceHeight); + void Destroy(); + + VASurfaceID InputSurface() const; + VASurfaceID Reconstructed(int index) const; + VABufferID CodedBuffer() const; + + // Puts the frame into the input surface as NV12, at the given size. + // A frame up to tolerance pixels larger is cropped, which needs no + // copy; one of another size, which raced with a change of size, is + // scaled. + bool Upload(const webrtc::VideoFrame & frame, int width, int height, int tolerance); + + bool AddBuffer(VABufferType type, void * data, size_t size); + bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size); + // The rate control, frame rate and HRD parameters, for constant + // bitrate with the given initial quantizer. + bool AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp); + + // Encodes the input surface with the parameter buffers added, and + // waits for it. + bool Encode(); + bool ReadOutput(std::vector & output); + // Drops the parameter buffers of the frame. + void DropBuffers(); + + bool Check(VAStatus status, const char * operation) const; + + private: + VaapiLibrary & library; + const VaapiFunctions & va; + const std::string name; + + VAConfigID config; + VAContextID context; + VASurfaceID surfaces[3]; + VABufferID codedBuffer; + std::vector frameBuffers; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h new file mode 100644 index 000000000..0e06ec5d1 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h @@ -0,0 +1,115 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_ENCODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_ENCODER_H_ + +#include "media/video/codec/EncoderOutputProcessor.h" +#include "media/video/codec/linux/VaapiEncodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" + +#include +#include + +#include +#include +#include + +namespace jni +{ + // Encodes H.265 Main with the GPU driver's VA-API encoder, as Intel and + // AMD drivers provide it on Linux. Experimental: it has not run on such a + // GPU yet, and is used only where it is turned on. + // + // It works as VaapiH264Encoder does: synchronous, one reference frame, + // P-frames only, the driver writing the parameter sets and slice headers + // from the parameters it is given. The block sizes and coding tools are + // those the driver says it takes, as FFmpeg's VA-API encoder chooses them. + // The frame size is kept to a multiple of the smallest coding block, so + // the coded picture is the frame and needs no cropping. There is no + // software H.265 to fall back to: anything that fails, including a driver + // that writes no parameter sets into key frames, is an error, upon which + // WebRTC switches to the next codec it negotiated. + class VaapiH265Encoder : public webrtc::VideoEncoder + { + public: + VaapiH265Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, const webrtc::SdpVideoFormat & format); + ~VaapiH265Encoder() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + private: + // What the driver says about the HEVC encoder it has. + void QueryCapabilities(); + bool Submit(bool idr); + + void FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const; + void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const; + void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const; + // The picture the current one refers to. + VAPictureHEVC Reference() const; + + private: + VaapiLibrary & library; + const VAEntrypoint entrypoint; + const std::string implementationName; + VaapiEncodeSession session; + + // The sizes, in log2 of luma samples, and the tools the driver + // takes. + uint32_t log2CtbSize; + uint32_t log2MinCbSize; + uint32_t log2MinTbSize; + uint32_t log2MaxTbSize; + uint32_t maxTransformDepthInter; + uint32_t maxTransformDepthIntra; + bool amp; + bool sao; + bool signDataHiding; + bool transformSkip; + bool cuQpDelta; + + bool initialized; + webrtc::VideoCodec codecSettings; + uint32_t widthInCtbs; + uint32_t heightInCtbs; + uint32_t bitrateBps; + uint32_t framerate; + bool ratesChanged; + + // The picture order count since the last IDR picture, and which of + // the reconstructed surfaces holds the current picture. + int32_t pictureOrder; + int current; + bool referenceValid; + + webrtc::EncodedImageCallback * callback; + + EncoderOutputProcessor outputProcessor; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiLibrary.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiLibrary.h index 849d32888..891767cc5 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiLibrary.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiLibrary.h @@ -56,6 +56,7 @@ namespace jni decltype(&::vaDeriveImage) DeriveImage = nullptr; decltype(&::vaCreateImage) CreateImage = nullptr; decltype(&::vaPutImage) PutImage = nullptr; + decltype(&::vaGetImage) GetImage = nullptr; decltype(&::vaDestroyImage) DestroyImage = nullptr; }; @@ -77,9 +78,17 @@ namespace jni VADisplay Display() const; - // The encode entrypoint the driver offers, full or low-power. + // The encode entrypoint the driver offers for H.264, full or + // low-power. VAEntrypoint Entrypoint() const; + // Whether the driver encodes the profile, 4:2:0 at constant + // bitrate, and with which entrypoint, full or low-power. + bool EncodeEntrypoint(VAProfile profile, VAEntrypoint * entrypoint) const; + + // Whether the driver decodes the profile into 4:2:0 surfaces. + bool Decodes(VAProfile profile) const; + // The driver, e.g. "Intel iHD driver for Intel(R) Gen Graphics". const std::string & Vendor() const; @@ -90,7 +99,7 @@ namespace jni bool Load(); bool OpenDisplay(); - bool SupportsEncoding(VADisplay display, VAEntrypoint * entrypoint) const; + bool SupportsEncoding(VADisplay display, VAProfile profile, VAEntrypoint * entrypoint) const; private: DynamicLibrary va; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h new file mode 100644 index 000000000..e666a2191 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_VIDEO_DECODER_FACTORY_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_VIDEO_DECODER_FACTORY_H_ + +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +#include +#include + +namespace jni +{ + // Creates VA-API decoders on the GPUs of Intel and AMD, for VP9 profile 0 + // where the driver decodes it, the format WebRTC's software decoder + // offers too. + class VaapiVideoDecoderFactory : public webrtc::VideoDecoderFactory + { + public: + // Returns a factory, or null if there is no VA-API driver, or one + // that decodes no codec offered here. + static std::unique_ptr Create(); + + ~VaapiVideoDecoderFactory() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + explicit VaapiVideoDecoderFactory(VaapiLibrary & library); + + private: + VaapiLibrary & library; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h index 5ac624849..972214e15 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h @@ -32,13 +32,15 @@ namespace jni // Creates VA-API encoders on the GPU whose driver encodes H.264, as Intel // and AMD drivers do. It offers H.264 in the profiles WebRTC's software // encoder offers too, and only with packetization mode 1: mode 0 needs - // each NAL unit to fit a packet. + // each NAL unit to fit a packet. With the experimental VA-API codecs + // turned on, it offers H.265 Main as well, and with hardware VP9 encoding + // turned on VP9 profile 0, where the driver encodes them. class VaapiVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: // Returns a factory, or null if no GPU driver encodes H.264 // through VA-API. - static std::unique_ptr Create(); + static std::unique_ptr Create(bool experimental, bool vp9Enabled); ~VaapiVideoEncoderFactory() override = default; @@ -47,10 +49,15 @@ namespace jni const webrtc::SdpVideoFormat & format) override; private: - explicit VaapiVideoEncoderFactory(VaapiLibrary & library); + VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, bool vp9, + VAEntrypoint vp9Entrypoint); private: VaapiLibrary & library; + const bool h265; + const VAEntrypoint h265Entrypoint; + const bool vp9; + const VAEntrypoint vp9Entrypoint; }; } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h new file mode 100644 index 000000000..263c23f89 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h @@ -0,0 +1,109 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_VP9_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_VP9_DECODER_H_ + +#include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/Vp9FrameHeaderParser.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace jni +{ + // Decodes VP9 profile 0 with the GPU driver's VA-API decoder, as Intel and + // AMD drivers provide it on Linux. Experimental: it has not run on such a + // GPU yet, and is used only where it is turned on. + // + // VA-API leaves the parsing of the stream to the application: the frame + // headers are parsed here, the driver decodes the rest, the compressed + // header with the probabilities included. Decoding is synchronous. The + // decoded frame is copied from its surface in NV12 and converted to I420, + // which is what WebRTC's frames hold. + // + // Whatever it cannot decode, such as another profile, spatial layers or + // an inter frame that changes the size, and whatever fails, returns + // WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so that libvpx takes over. + class VaapiVp9Decoder : public webrtc::VideoDecoder + { + public: + explicit VaapiVp9Decoder(VaapiLibrary & library); + ~VaapiVp9Decoder() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // Creates the configuration, the context and the surfaces for + // frames of the size. + bool CreateSession(uint32_t width, uint32_t height); + void DestroySession(); + + // Decodes one frame of the image into the surface. + bool DecodeFrame(const Vp9FrameHeader & header, std::span frame, int surface); + // Hands the frame in the surface to WebRTC. + bool Output(int surface, const webrtc::EncodedImage & image, int64_t renderTimeMs); + // A surface that no reference slot holds. + int FreeSurface() const; + + bool Check(VAStatus status, const char * operation) const; + + private: + VaapiLibrary & library; + const VaapiFunctions & va; + const std::string implementationName; + + webrtc::DecodedImageCallback * callback; + + Vp9FrameHeaderParser parser; + + VAConfigID config; + VAContextID context; + std::vector surfaces; + uint32_t width; + uint32_t height; + + // The image a decoded surface is copied into, where the driver + // copies surfaces at all. + VAImage downloadImage; + + // The surface each of the eight reference slots holds, -1 for none. + std::array slots; + + // Whether a key frame has been decoded, and whether anything since + // then failed for good. + bool started; + bool failed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h new file mode 100644 index 000000000..9bcfb3aa2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_VP9_ENCODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_VP9_ENCODER_H_ + +#include "media/video/codec/EncoderOutputProcessor.h" +#include "media/video/codec/linux/VaapiEncodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" + +#include +#include + +#include +#include +#include + +namespace jni +{ + // Encodes VP9 profile 0 with the GPU driver's VA-API encoder, as Intel + // drivers provide it on Linux. Like hardware VP9 encoding on Windows, it + // has not run on such a GPU yet, and is used only where it is turned on. + // + // It works as the H.265 one does: synchronous, one reference frame, inter + // frames only, the driver writing the uncompressed frame header from the + // parameters it is given, as FFmpeg's VA-API encoder sets them without + // B-frames. Every frame refreshes all eight reference slots, so that the + // next one can refer to it as the last frame whichever slot it names. + // Anything that fails is an error, upon which libvpx takes over. + class VaapiVp9Encoder : public webrtc::VideoEncoder + { + public: + VaapiVp9Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, const webrtc::SdpVideoFormat & format); + ~VaapiVp9Encoder() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + private: + bool Submit(bool keyFrame); + + void FillSequence(VAEncSequenceParameterBufferVP9 & sequence) const; + void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const; + + private: + VaapiLibrary & library; + const VAEntrypoint entrypoint; + const std::string implementationName; + VaapiEncodeSession session; + + bool initialized; + webrtc::VideoCodec codecSettings; + uint32_t bitrateBps; + uint32_t framerate; + bool ratesChanged; + + // Which of the reconstructed surfaces holds the current frame. + int current; + bool referenceValid; + + webrtc::EncodedImageCallback * callback; + + EncoderOutputProcessor outputProcessor; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Vp9FrameHeaderParser.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Vp9FrameHeaderParser.h new file mode 100644 index 000000000..1a63dc3cd --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Vp9FrameHeaderParser.h @@ -0,0 +1,149 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VP9_FRAME_HEADER_PARSER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VP9_FRAME_HEADER_PARSER_H_ + +#include +#include +#include +#include +#include + +namespace jni +{ + // What a hardware decoder needs to know about one segment of a VP9 frame: + // its quantizer scales and loop filter levels, worked out from the frame + // and its segmentation as libvpx does. + struct Vp9Segment + { + bool referenceEnabled = false; + uint8_t reference = 0; + bool skipped = false; + + // By reference frame (intra, last, golden, altref) and mode delta + // (zero motion vector or not). + uint8_t filterLevel[4][2] = {}; + + int16_t lumaDcScale = 0; + int16_t lumaAcScale = 0; + int16_t chromaDcScale = 0; + int16_t chromaAcScale = 0; + }; + + // The uncompressed header of a VP9 frame, as section 6.2 of the VP9 + // bitstream specification defines it, with what earlier frames left that + // it depends on resolved: the size taken from a reference frame, the + // loop filter deltas and the segmentation that carry over. + struct Vp9FrameHeader + { + uint8_t profile = 0; + uint8_t bitDepth = 8; + bool subsamplingX = true; + bool subsamplingY = true; + + bool showExistingFrame = false; + uint8_t frameToShow = 0; + + bool keyFrame = false; + bool showFrame = false; + bool errorResilient = false; + bool intraOnly = false; + uint8_t resetFrameContext = 0; + uint8_t refreshFrameFlags = 0; + + // The slots of the last, golden and altref frames, and their sign + // bias. + std::array referenceSlots = {}; + std::array referenceSignBias = {}; + + uint32_t width = 0; + uint32_t height = 0; + + bool allowHighPrecisionMv = false; + // The interpolation filter as coded, 0 to 3, or 4 for switchable, + // which is what VA-API takes. + uint8_t interpolationFilter = 0; + + bool refreshFrameContext = false; + bool frameParallelDecoding = false; + uint8_t frameContextIndex = 0; + + uint8_t filterLevel = 0; + uint8_t sharpness = 0; + + uint8_t baseQIndex = 0; + bool lossless = false; + + bool segmentationEnabled = false; + bool segmentationUpdateMap = false; + bool segmentationTemporalUpdate = false; + std::array segmentationTreeProbs = {}; + std::array segmentationPredProbs = {}; + std::array segments = {}; + + uint8_t log2TileColumns = 0; + uint8_t log2TileRows = 0; + + // The sizes of the uncompressed and compressed headers, in bytes. + uint32_t uncompressedHeaderSize = 0; + uint32_t compressedHeaderSize = 0; + }; + + // Parses the uncompressed headers of the frames of a VP9 stream, in the + // order they are decoded, keeping what carries over from one to the + // next. Profile 0 only, which is all a VA-API decoder is asked for here. + class Vp9FrameHeaderParser + { + public: + Vp9FrameHeaderParser(); + + // Forgets what earlier frames left, as before a new stream. + void Reset(); + + // Parses the header of the frame and takes over what it changes. + // Returns nothing if the frame is not valid profile 0 VP9, or + // refers to a reference frame that is not there. + std::optional Parse(std::span frame); + + // The frames of a superframe, or the frame itself, as spans of + // the data. + static std::vector> SplitSuperframe(std::span data); + + private: + // What the specification calls setup_past_independence. + void SetupPastIndependence(); + void ComputeSegments(Vp9FrameHeader & header, int8_t yDcDelta, int8_t uvDcDelta, int8_t uvAcDelta) const; + + private: + std::array loopFilterRefDeltas; + std::array loopFilterModeDeltas; + bool loopFilterDeltaEnabled; + + std::array segmentationTreeProbs; + std::array segmentationPredProbs; + bool segmentationAbsoluteValues; + bool featureEnabled[8][4]; + int16_t featureData[8][4]; + + // The size of the frame in each reference slot, zero where there + // is none. + std::array slotWidths; + std::array slotHeights; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp index 5e337059d..bb74c835a 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp @@ -21,43 +21,58 @@ namespace jni { - bool IsHardwareVp9EncoderEnabled() + namespace { - JNIEnv * env = AttachCurrentThread(); + // Whether the Java system property is "true", as Boolean.getBoolean has + // it; false where Java cannot be asked. + bool JavaBooleanProperty(const char * property) + { + JNIEnv * env = AttachCurrentThread(); - // An exception pending is someone else's to handle, and no JNI call - // may be made meanwhile. - if (env == nullptr || env->ExceptionCheck()) { - return false; - } - if (env->PushLocalFrame(4) != JNI_OK) { - env->ExceptionClear(); - return false; - } + // An exception pending is someone else's to handle, and no JNI call + // may be made meanwhile. + if (env == nullptr || env->ExceptionCheck()) { + return false; + } + if (env->PushLocalFrame(4) != JNI_OK) { + env->ExceptionClear(); + return false; + } - bool enabled = false; + bool enabled = false; - // java.lang.Boolean comes from the bootstrap class loader, which any - // thread reaches, one without Java frames as well. - jclass booleanClass = env->FindClass("java/lang/Boolean"); + // java.lang.Boolean comes from the bootstrap class loader, which + // any thread reaches, one without Java frames as well. + jclass booleanClass = env->FindClass("java/lang/Boolean"); - if (booleanClass != nullptr) { - jmethodID getBoolean = env->GetStaticMethodID(booleanClass, "getBoolean", "(Ljava/lang/String;)Z"); - jstring name = getBoolean != nullptr ? env->NewStringUTF(kHardwareVp9EncoderProperty) : nullptr; + if (booleanClass != nullptr) { + jmethodID getBoolean = env->GetStaticMethodID(booleanClass, "getBoolean", "(Ljava/lang/String;)Z"); + jstring name = getBoolean != nullptr ? env->NewStringUTF(property) : nullptr; - if (name != nullptr) { - enabled = env->CallStaticBooleanMethod(booleanClass, getBoolean, name) == JNI_TRUE; + if (name != nullptr) { + enabled = env->CallStaticBooleanMethod(booleanClass, getBoolean, name) == JNI_TRUE; + } } - } - // Raised here, so cleared here. - if (env->ExceptionCheck()) { - env->ExceptionClear(); - enabled = false; + // Raised here, so cleared here. + if (env->ExceptionCheck()) { + env->ExceptionClear(); + enabled = false; + } + + env->PopLocalFrame(nullptr); + + return enabled; } + } - env->PopLocalFrame(nullptr); + bool IsHardwareVp9EncoderEnabled() + { + return JavaBooleanProperty(kHardwareVp9EncoderProperty); + } - return enabled; + bool IsExperimentalVaapiEnabled() + { + return JavaBooleanProperty(kExperimentalVaapiProperty); } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index 549a703d7..7f144f206 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -14,8 +14,10 @@ * limitations under the License. */ +#include "media/video/codec/HardwareCodecOptions.h" #include "media/video/codec/HardwareVideoDecoderFactory.h" #include "media/video/codec/HardwareVideoEncoderFactory.h" +#include "media/video/codec/linux/VaapiVideoDecoderFactory.h" #include "media/video/codec/linux/VaapiVideoEncoderFactory.h" #include "media/video/codec/nvdec/NvdecVideoDecoderFactory.h" #include "media/video/codec/nvenc/NvencVideoEncoderFactory.h" @@ -31,7 +33,12 @@ namespace jni if (auto nvenc = NvencVideoEncoderFactory::Create()) { factories.push_back(std::move(nvenc)); } - if (auto vaapi = VaapiVideoEncoderFactory::Create()) { + // The VA-API codecs beyond H.264 only if asked for, H.265 and VP9 + // each with its own switch: they have not been tested on an Intel or + // AMD GPU yet. + auto vaapi = VaapiVideoEncoderFactory::Create(IsExperimentalVaapiEnabled(), IsHardwareVp9EncoderEnabled()); + + if (vaapi) { factories.push_back(std::move(vaapi)); } @@ -42,11 +49,16 @@ namespace jni { std::vector> factories; - // NVDEC, on NVIDIA GPUs. Decoding with VA-API, for the GPUs of Intel - // and AMD, is to come. + // NVDEC on NVIDIA GPUs; VA-API for the GPUs of Intel and AMD, only if + // asked for, since it has not been tested on such a GPU yet. if (auto nvdec = NvdecVideoDecoderFactory::Create()) { factories.push_back(std::move(nvdec)); } + if (IsExperimentalVaapiEnabled()) { + if (auto vaapi = VaapiVideoDecoderFactory::Create()) { + factories.push_back(std::move(vaapi)); + } + } return factories; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp new file mode 100644 index 000000000..adda0ba2a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp @@ -0,0 +1,325 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiEncodeSession.h" + +#include "api/video/i420_buffer.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert_from.h" + +#include + +namespace jni +{ + namespace + { + // How many seconds of video the rate control may buffer, as a + // multiple of the bitrate. + constexpr uint32_t kHrdBufferNumerator = 3; + constexpr uint32_t kHrdBufferDenominator = 2; + } + + VaapiEncodeSession::VaapiEncodeSession(VaapiLibrary & library, std::string name) : + library(library), + va(library.Api()), + name(std::move(name)), + config(VA_INVALID_ID), + context(VA_INVALID_ID), + surfaces{ VA_INVALID_SURFACE, VA_INVALID_SURFACE, VA_INVALID_SURFACE }, + codedBuffer(VA_INVALID_ID) + { + } + + VaapiEncodeSession::~VaapiEncodeSession() + { + Destroy(); + } + + bool VaapiEncodeSession::Check(VAStatus status, const char * operation) const + { + if (status == VA_STATUS_SUCCESS) { + return true; + } + + RTC_LOG(LS_WARNING) << name << ": " << operation << " failed: " << library.ErrorString(status); + + return false; + } + + bool VaapiEncodeSession::Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t surfaceWidth, + uint32_t surfaceHeight) + { + Destroy(); + + VADisplay display = library.Display(); + + VAConfigAttrib attributes[2] = {}; + attributes[0].type = VAConfigAttribRTFormat; + attributes[0].value = VA_RT_FORMAT_YUV420; + attributes[1].type = VAConfigAttribRateControl; + attributes[1].value = VA_RC_CBR; + + if (!Check(va.CreateConfig(display, profile, entrypoint, attributes, 2, &config), "creating the configuration")) { + config = VA_INVALID_ID; + return false; + } + + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces, 3, nullptr, 0), + "creating surfaces")) + { + for (VASurfaceID & surface : surfaces) { + surface = VA_INVALID_SURFACE; + } + + return false; + } + + if (!Check(va.CreateContext(display, config, surfaceWidth, surfaceHeight, VA_PROGRESSIVE, surfaces, 3, &context), + "creating the context")) + { + context = VA_INVALID_ID; + return false; + } + + // Room for a frame that does not compress at all. + const unsigned int codedSize = surfaceWidth * surfaceHeight * 3 / 2 + 64 * 1024; + + if (!Check(va.CreateBuffer(display, context, VAEncCodedBufferType, codedSize, 1, nullptr, &codedBuffer), + "creating the output buffer")) + { + codedBuffer = VA_INVALID_ID; + return false; + } + + return true; + } + + void VaapiEncodeSession::Destroy() + { + VADisplay display = library.Display(); + + DropBuffers(); + + if (codedBuffer != VA_INVALID_ID) { + va.DestroyBuffer(display, codedBuffer); + codedBuffer = VA_INVALID_ID; + } + if (context != VA_INVALID_ID) { + va.DestroyContext(display, context); + context = VA_INVALID_ID; + } + if (surfaces[0] != VA_INVALID_SURFACE) { + va.DestroySurfaces(display, surfaces, 3); + } + + for (VASurfaceID & surface : surfaces) { + surface = VA_INVALID_SURFACE; + } + + if (config != VA_INVALID_ID) { + va.DestroyConfig(display, config); + config = VA_INVALID_ID; + } + } + + VASurfaceID VaapiEncodeSession::InputSurface() const + { + return surfaces[0]; + } + + VASurfaceID VaapiEncodeSession::Reconstructed(int index) const + { + return surfaces[1 + index]; + } + + VABufferID VaapiEncodeSession::CodedBuffer() const + { + return codedBuffer; + } + + bool VaapiEncodeSession::Upload(const webrtc::VideoFrame & frame, int width, int height, int tolerance) + { + webrtc::scoped_refptr i420 = frame.video_frame_buffer()->ToI420(); + + if (!i420) { + return false; + } + + const bool cropped = i420->width() >= width && i420->width() - width <= tolerance + && i420->height() >= height && i420->height() - height <= tolerance; + + if (!cropped) { + webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); + scaled->ScaleFrom(*i420); + i420 = scaled; + } + + VADisplay display = library.Display(); + VAImage image = {}; + image.image_id = VA_INVALID_ID; + + // Writing into the surface directly where the driver allows it, + // otherwise into an image that is copied into the surface. + bool derived = va.DeriveImage(display, InputSurface(), &image) == VA_STATUS_SUCCESS + && image.format.fourcc == VA_FOURCC_NV12; + + if (!derived) { + if (image.image_id != VA_INVALID_ID) { + va.DestroyImage(display, image.image_id); + } + + VAImageFormat format = {}; + format.fourcc = VA_FOURCC_NV12; + format.byte_order = VA_LSB_FIRST; + format.bits_per_pixel = 12; + + if (!Check(va.CreateImage(display, &format, width, height, &image), "creating an image")) { + return false; + } + } + + void * data = nullptr; + bool uploaded = Check(va.MapBuffer(display, image.buf, &data), "mapping an image"); + + if (uploaded) { + uint8_t * base = static_cast(data); + + libyuv::I420ToNV12(i420->DataY(), i420->StrideY(), i420->DataU(), i420->StrideU(), + i420->DataV(), i420->StrideV(), + base + image.offsets[0], static_cast(image.pitches[0]), + base + image.offsets[1], static_cast(image.pitches[1]), + width, height); + + va.UnmapBuffer(display, image.buf); + + if (!derived) { + uploaded = Check(va.PutImage(display, InputSurface(), image.image_id, 0, 0, width, height, + 0, 0, width, height), "copying an image"); + } + } + + va.DestroyImage(display, image.image_id); + + return uploaded; + } + + bool VaapiEncodeSession::AddBuffer(VABufferType type, void * data, size_t size) + { + VABufferID buffer = VA_INVALID_ID; + + if (!Check(va.CreateBuffer(library.Display(), context, type, static_cast(size), 1, data, + &buffer), "creating a parameter buffer")) + { + return false; + } + + frameBuffers.push_back(buffer); + + return true; + } + + bool VaapiEncodeSession::AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size) + { + // A VAEncMiscParameterBuffer: the type, followed by the parameter. + std::vector buffer(sizeof(VAEncMiscParameterBuffer) + size); + + auto * header = reinterpret_cast(buffer.data()); + header->type = type; + std::memcpy(header->data, data, size); + + return AddBuffer(VAEncMiscParameterBufferType, buffer.data(), buffer.size()); + } + + bool VaapiEncodeSession::AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp) + { + VAEncMiscParameterRateControl rateControl = {}; + rateControl.bits_per_second = bitrateBps; + rateControl.target_percentage = 100; + rateControl.window_size = 1000; + rateControl.initial_qp = initialQp; + + VAEncMiscParameterFrameRate frameRate = {}; + // The numerator in the low 16 bits, the denominator in the high. + frameRate.framerate = framerate | (1u << 16); + + VAEncMiscParameterHRD hrd = {}; + hrd.buffer_size = static_cast( + static_cast(bitrateBps) * kHrdBufferNumerator / kHrdBufferDenominator); + hrd.initial_buffer_fullness = hrd.buffer_size / 2; + + return AddMiscParameter(VAEncMiscParameterTypeRateControl, &rateControl, sizeof(rateControl)) + && AddMiscParameter(VAEncMiscParameterTypeFrameRate, &frameRate, sizeof(frameRate)) + && AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); + } + + bool VaapiEncodeSession::Encode() + { + VADisplay display = library.Display(); + + if (!Check(va.BeginPicture(display, context, InputSurface()), "beginning a picture")) { + return false; + } + + bool rendered = Check(va.RenderPicture(display, context, frameBuffers.data(), + static_cast(frameBuffers.size())), "rendering a picture"); + + // A picture that was begun has to be ended. + bool ended = Check(va.EndPicture(display, context), "ending a picture"); + + return rendered && ended && Check(va.SyncSurface(display, InputSurface()), "waiting for a picture"); + } + + bool VaapiEncodeSession::ReadOutput(std::vector & output) + { + VADisplay display = library.Display(); + void * data = nullptr; + + if (!Check(va.MapBuffer(display, codedBuffer, &data), "mapping the output")) { + return false; + } + + bool complete = true; + + for (auto * segment = static_cast(data); segment != nullptr; + segment = static_cast(segment->next)) + { + if (segment->status & VA_CODED_BUF_STATUS_SLICE_OVERFLOW_MASK) { + complete = false; + } + + const uint8_t * bytes = static_cast(segment->buf); + output.insert(output.end(), bytes, bytes + segment->size); + } + + va.UnmapBuffer(display, codedBuffer); + + if (!complete || output.empty()) { + RTC_LOG(LS_WARNING) << name << ": a frame did not fit the output buffer"; + return false; + } + + return true; + } + + void VaapiEncodeSession::DropBuffers() + { + for (VABufferID buffer : frameBuffers) { + va.DestroyBuffer(library.Display(), buffer); + } + + frameBuffers.clear(); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp new file mode 100644 index 000000000..b341e6960 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp @@ -0,0 +1,462 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiH265Encoder.h" + +#include "api/video/encoded_image.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +#include + +namespace jni +{ + namespace + { + // Key frames come when WebRTC asks for them; this only bounds how + // long a stream may go without one if it never does. + constexpr uint32_t kIntraPeriod = 3000; + + // H.265 slice types (7.4.7.1) and NAL unit types (table 7-1). + constexpr uint8_t kSliceTypeP = 1; + constexpr uint8_t kSliceTypeI = 2; + constexpr uint8_t kNalTrailR = 1; + constexpr uint8_t kNalIdrWRadl = 19; + + // VA-API's picture coding types. + constexpr uint32_t kCodingTypeI = 1; + constexpr uint32_t kCodingTypeP = 2; + + constexpr uint32_t kInitialQp = 26; + + // The lowest H.265 level whose limits a stream fits in, going by its + // picture size and luma sample rate, per table A.8 of the standard; + // general_level_idc is 30 times the level. + uint8_t LevelIdc(uint32_t pictureSize, uint32_t framerate) + { + struct Level + { + uint8_t idc; + uint64_t maxLumaPs; + uint64_t maxLumaSr; + }; + + static const Level levels[] = { + { 93, 983040, 33177600 }, + { 120, 2228224, 66846720 }, + { 123, 2228224, 133693440 }, + { 150, 8912896, 267386880 }, + { 153, 8912896, 534773760 }, + { 156, 8912896, 1069547520 } + }; + + const uint64_t sampleRate = static_cast(pictureSize) * framerate; + + for (const Level & level : levels) { + if (pictureSize <= level.maxLumaPs && sampleRate <= level.maxLumaSr) { + return level.idc; + } + } + + return 156; + } + + void InvalidatePicture(VAPictureHEVC & picture) + { + picture = {}; + picture.picture_id = VA_INVALID_SURFACE; + picture.flags = VA_PICTURE_HEVC_INVALID; + } + + // A coding tool is used where the driver allows it. + bool Allowed(uint32_t feature) + { + return feature != VA_FEATURE_NOT_SUPPORTED; + } + } + + VaapiH265Encoder::VaapiH265Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, + const webrtc::SdpVideoFormat & format) : + library(library), + entrypoint(entrypoint), + implementationName("VA-API (" + library.Vendor() + ")"), + session(library, "VA-API H.265"), + // FFmpeg's choice for a driver that does not say: 32x32 coding tree + // blocks, coding blocks of at least 16x16, transform blocks of 4x4 to + // 32x32 and three levels of transform hierarchy. + log2CtbSize(5), + log2MinCbSize(4), + log2MinTbSize(2), + log2MaxTbSize(5), + maxTransformDepthInter(3), + maxTransformDepthIntra(3), + amp(true), + sao(true), + signDataHiding(false), + transformSkip(false), + cuQpDelta(true), + initialized(false), + codecSettings(), + widthInCtbs(0), + heightInCtbs(0), + bitrateBps(0), + framerate(30), + ratesChanged(false), + pictureOrder(0), + current(0), + referenceValid(false), + callback(nullptr), + outputProcessor(webrtc::kVideoCodecH265, format) + { + QueryCapabilities(); + } + + VaapiH265Encoder::~VaapiH265Encoder() + { + Release(); + } + + void VaapiH265Encoder::QueryCapabilities() + { + VAConfigAttrib attributes[2] = {}; + attributes[0].type = VAConfigAttribEncHEVCBlockSizes; + attributes[1].type = VAConfigAttribEncHEVCFeatures; + + if (library.Api().GetConfigAttributes(library.Display(), VAProfileHEVCMain, entrypoint, attributes, 2) != + VA_STATUS_SUCCESS) + { + return; + } + + if (attributes[0].value != VA_ATTRIB_NOT_SUPPORTED) { + VAConfigAttribValEncHEVCBlockSizes sizes; + sizes.value = attributes[0].value; + + log2CtbSize = sizes.bits.log2_max_coding_tree_block_size_minus3 + 3; + log2MinCbSize = sizes.bits.log2_min_luma_coding_block_size_minus3 + 3; + log2MinTbSize = sizes.bits.log2_min_luma_transform_block_size_minus2 + 2; + log2MaxTbSize = sizes.bits.log2_max_luma_transform_block_size_minus2 + 2; + maxTransformDepthInter = sizes.bits.max_max_transform_hierarchy_depth_inter; + maxTransformDepthIntra = sizes.bits.max_max_transform_hierarchy_depth_intra; + } + + if (attributes[1].value != VA_ATTRIB_NOT_SUPPORTED) { + VAConfigAttribValEncHEVCFeatures features; + features.value = attributes[1].value; + + amp = Allowed(features.bits.amp); + sao = Allowed(features.bits.sao); + signDataHiding = Allowed(features.bits.sign_data_hiding); + transformSkip = Allowed(features.bits.transform_skip); + cuQpDelta = Allowed(features.bits.cu_qp_delta); + } + } + + int32_t VaapiH265Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + if (settings == nullptr || settings->width == 0 || settings->height == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (settings->numberOfSimulcastStreams > 1) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + + // The coded picture is kept to whole coding blocks, so that it is the + // frame and needs no cropping. WebRTC is asked for frames of that + // alignment, but the first frames of a source can come before it + // knows, so the encoder takes the aligned size below and drops the + // rows and columns beyond it. + const uint32_t alignment = 1u << log2MinCbSize; + + if (settings->width < alignment || settings->height < alignment) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + Release(); + + const uint32_t ctbSize = 1u << log2CtbSize; + + codecSettings = *settings; + codecSettings.width -= codecSettings.width % alignment; + codecSettings.height -= codecSettings.height % alignment; + widthInCtbs = (codecSettings.width + ctbSize - 1) / ctbSize; + heightInCtbs = (codecSettings.height + ctbSize - 1) / ctbSize; + bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; + framerate = std::max(1u, codecSettings.maxFramerate); + ratesChanged = true; + pictureOrder = 0; + current = 0; + referenceValid = false; + outputProcessor.Reset(); + + // Whole coding tree blocks, which the driver may write all of. + if (!session.Create(VAProfileHEVCMain, entrypoint, widthInCtbs << log2CtbSize, heightInCtbs << log2CtbSize)) { + Release(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + initialized = true; + + RTC_LOG(LS_INFO) << implementationName << " H.265 initialized: " << codecSettings.width << "x" + << codecSettings.height << " at " << bitrateBps << " bps, CTB " << ctbSize << ", min CB " << alignment; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiH265Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiH265Encoder::Release() + { + session.Destroy(); + + initialized = false; + referenceValid = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiH265Encoder::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized || callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + bool idr = !referenceValid; + + if (frameTypes != nullptr) { + idr |= std::any_of(frameTypes->begin(), frameTypes->end(), [](webrtc::VideoFrameType type) { + return type == webrtc::VideoFrameType::kVideoFrameKey; + }); + } + + if (idr) { + pictureOrder = 0; + } + + const int alignment = 1 << log2MinCbSize; + std::vector output; + + const bool encoded = session.Upload(frame, static_cast(codecSettings.width), + static_cast(codecSettings.height), alignment - 1) && Submit(idr) && session.ReadOutput(output); + + session.DropBuffers(); + + if (!encoded) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + webrtc::EncodedImage image; + image._encodedWidth = codecSettings.width; + image._encodedHeight = codecSettings.height; + image.SetRtpTimestamp(frame.rtp_timestamp()); + image.capture_time_ms_ = frame.render_time_ms(); + image.ntp_time_ms_ = frame.ntp_time_ms(); + image.rotation_ = frame.rotation(); + + webrtc::CodecSpecificInfo info; + + // A driver that writes no parameter sets into key frames makes a + // stream nothing can decode. + if (!outputProcessor.Process(output, idr, image, info)) { + RTC_LOG(LS_WARNING) << implementationName << " H.265: the driver wrote no parameter sets"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // The frame just encoded is the reference of the next one. + referenceValid = true; + current = 1 - current; + pictureOrder++; + ratesChanged = false; + + callback->OnEncodedImage(image, &info); + + return WEBRTC_VIDEO_CODEC_OK; + } + + bool VaapiH265Encoder::Submit(bool idr) + { + if (idr) { + VAEncSequenceParameterBufferHEVC sequence = {}; + FillSequence(sequence); + + if (!session.AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { + return false; + } + } + + if ((idr || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, kInitialQp)) { + return false; + } + + VAEncPictureParameterBufferHEVC picture = {}; + FillPicture(picture, idr); + + VAEncSliceParameterBufferHEVC slice = {}; + FillSlice(slice, idr); + + return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) + && session.AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice)) + && session.Encode(); + } + + void VaapiH265Encoder::FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const + { + // Main profile, main tier. + sequence.general_profile_idc = 1; + sequence.general_level_idc = LevelIdc(codecSettings.width * codecSettings.height, framerate); + sequence.general_tier_flag = 0; + sequence.intra_period = kIntraPeriod; + sequence.intra_idr_period = kIntraPeriod; + sequence.ip_period = 1; + sequence.bits_per_second = bitrateBps; + sequence.pic_width_in_luma_samples = static_cast(codecSettings.width); + sequence.pic_height_in_luma_samples = static_cast(codecSettings.height); + + sequence.seq_fields.bits.chroma_format_idc = 1; + sequence.seq_fields.bits.amp_enabled_flag = amp ? 1 : 0; + sequence.seq_fields.bits.sample_adaptive_offset_enabled_flag = sao ? 1 : 0; + // No temporal motion vector prediction, which would need a collocated + // picture to be named. + sequence.seq_fields.bits.sps_temporal_mvp_enabled_flag = 0; + sequence.seq_fields.bits.low_delay_seq = 1; + + sequence.log2_min_luma_coding_block_size_minus3 = static_cast(log2MinCbSize - 3); + sequence.log2_diff_max_min_luma_coding_block_size = static_cast(log2CtbSize - log2MinCbSize); + sequence.log2_min_transform_block_size_minus2 = static_cast(log2MinTbSize - 2); + sequence.log2_diff_max_min_transform_block_size = static_cast(log2MaxTbSize - log2MinTbSize); + sequence.max_transform_hierarchy_depth_inter = static_cast(maxTransformDepthInter); + sequence.max_transform_hierarchy_depth_intra = static_cast(maxTransformDepthIntra); + + sequence.vui_parameters_present_flag = 1; + sequence.vui_fields.bits.vui_timing_info_present_flag = 1; + sequence.vui_num_units_in_tick = 1; + sequence.vui_time_scale = framerate; + } + + VAPictureHEVC VaapiH265Encoder::Reference() const + { + VAPictureHEVC reference = {}; + reference.picture_id = session.Reconstructed(1 - current); + reference.pic_order_cnt = pictureOrder - 1; + // A short-term reference before the current picture. + reference.flags = VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE; + + return reference; + } + + void VaapiH265Encoder::FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const + { + picture.decoded_curr_pic.picture_id = session.Reconstructed(current); + picture.decoded_curr_pic.pic_order_cnt = pictureOrder; + picture.decoded_curr_pic.flags = 0; + + for (VAPictureHEVC & reference : picture.reference_frames) { + InvalidatePicture(reference); + } + + if (!idr) { + picture.reference_frames[0] = Reference(); + } + + picture.coded_buf = session.CodedBuffer(); + // No collocated picture: there is no temporal motion vector + // prediction. + picture.collocated_ref_pic_index = 0xFF; + picture.last_picture = 0; + picture.pic_init_qp = kInitialQp; + picture.diff_cu_qp_delta_depth = 0; + picture.num_ref_idx_l0_default_active_minus1 = 0; + picture.num_ref_idx_l1_default_active_minus1 = 0; + picture.slice_pic_parameter_set_id = 0; + picture.nal_unit_type = idr ? kNalIdrWRadl : kNalTrailR; + + picture.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; + picture.pic_fields.bits.coding_type = idr ? kCodingTypeI : kCodingTypeP; + picture.pic_fields.bits.reference_pic_flag = 1; + picture.pic_fields.bits.sign_data_hiding_enabled_flag = signDataHiding ? 1 : 0; + picture.pic_fields.bits.transform_skip_enabled_flag = transformSkip ? 1 : 0; + // Needed for the rate control to change the QP within a picture. + picture.pic_fields.bits.cu_qp_delta_enabled_flag = cuQpDelta ? 1 : 0; + } + + void VaapiH265Encoder::FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const + { + slice.slice_segment_address = 0; + slice.num_ctu_in_slice = widthInCtbs * heightInCtbs; + slice.slice_type = idr ? kSliceTypeI : kSliceTypeP; + slice.slice_pic_parameter_set_id = 0; + slice.num_ref_idx_l0_active_minus1 = 0; + slice.num_ref_idx_l1_active_minus1 = 0; + + for (VAPictureHEVC & reference : slice.ref_pic_list0) { + InvalidatePicture(reference); + } + for (VAPictureHEVC & reference : slice.ref_pic_list1) { + InvalidatePicture(reference); + } + + if (!idr) { + slice.ref_pic_list0[0] = Reference(); + } + + slice.max_num_merge_cand = 5; + slice.slice_qp_delta = 0; + + slice.slice_fields.bits.last_slice_of_pic_flag = 1; + slice.slice_fields.bits.slice_sao_luma_flag = sao ? 1 : 0; + slice.slice_fields.bits.slice_sao_chroma_flag = sao ? 1 : 0; + } + + void VaapiH265Encoder::SetRates(const RateControlParameters & parameters) + { + const uint32_t bitrate = parameters.bitrate.get_sum_bps(); + + // A zero bitrate pauses the stream; the encoder gets no frames then. + if (bitrate == 0) { + return; + } + + bitrateBps = bitrate; + + if (parameters.framerate_fps >= 1.0) { + framerate = static_cast(parameters.framerate_fps + 0.5); + } + + // Applied with the next frame. + ratesChanged = true; + } + + webrtc::VideoEncoder::EncoderInfo VaapiH265Encoder::GetEncoderInfo() const + { + EncoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + info.supports_native_handle = false; + info.supports_simulcast = false; + info.scaling_settings = outputProcessor.GetScalingSettings(); + // Whole coding blocks, so that the coded picture is the frame. + info.requested_resolution_alignment = 1 << log2MinCbSize; + + return info; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiLibrary.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiLibrary.cpp index f97cd7e2f..a54b3362a 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiLibrary.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiLibrary.cpp @@ -90,6 +90,7 @@ namespace jni && va.Resolve("vaDeriveImage", api.DeriveImage) && va.Resolve("vaCreateImage", api.CreateImage) && va.Resolve("vaPutImage", api.PutImage) + && va.Resolve("vaGetImage", api.GetImage) && va.Resolve("vaDestroyImage", api.DestroyImage); if (!resolved) { @@ -132,7 +133,7 @@ namespace jni if (nodeDisplay != nullptr && api.Initialize(nodeDisplay, &major, &minor) == VA_STATUS_SUCCESS) { VAEntrypoint nodeEntrypoint; - if (SupportsEncoding(nodeDisplay, &nodeEntrypoint)) { + if (SupportsEncoding(nodeDisplay, VAProfileH264ConstrainedBaseline, &nodeEntrypoint)) { fd = nodeFd; display = nodeDisplay; entrypoint = nodeEntrypoint; @@ -152,7 +153,7 @@ namespace jni return false; } - bool VaapiLibrary::SupportsEncoding(VADisplay nodeDisplay, VAEntrypoint * supported) const + bool VaapiLibrary::SupportsEncoding(VADisplay nodeDisplay, VAProfile profile, VAEntrypoint * supported) const { const int maxEntrypoints = api.MaxNumEntrypoints(nodeDisplay); @@ -163,7 +164,7 @@ namespace jni std::vector entrypoints(maxEntrypoints); int count = 0; - if (api.QueryConfigEntrypoints(nodeDisplay, VAProfileH264ConstrainedBaseline, entrypoints.data(), + if (api.QueryConfigEntrypoints(nodeDisplay, profile, entrypoints.data(), &count) != VA_STATUS_SUCCESS) { return false; @@ -183,7 +184,7 @@ namespace jni attributes[0].type = VAConfigAttribRTFormat; attributes[1].type = VAConfigAttribRateControl; - if (api.GetConfigAttributes(nodeDisplay, VAProfileH264ConstrainedBaseline, candidate, attributes, 2) != + if (api.GetConfigAttributes(nodeDisplay, profile, candidate, attributes, 2) != VA_STATUS_SUCCESS) { continue; @@ -218,6 +219,42 @@ namespace jni return entrypoint; } + bool VaapiLibrary::EncodeEntrypoint(VAProfile profile, VAEntrypoint * supported) const + { + return SupportsEncoding(display, profile, supported); + } + + bool VaapiLibrary::Decodes(VAProfile profile) const + { + const int maxEntrypoints = api.MaxNumEntrypoints(display); + + if (maxEntrypoints <= 0) { + return false; + } + + std::vector entrypoints(maxEntrypoints); + int count = 0; + + if (api.QueryConfigEntrypoints(display, profile, entrypoints.data(), &count) != VA_STATUS_SUCCESS) { + return false; + } + + entrypoints.resize(count); + + if (std::find(entrypoints.begin(), entrypoints.end(), VAEntrypointVLD) == entrypoints.end()) { + return false; + } + + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribRTFormat; + + if (api.GetConfigAttributes(display, profile, VAEntrypointVLD, &attribute, 1) != VA_STATUS_SUCCESS) { + return false; + } + + return attribute.value != VA_ATTRIB_NOT_SUPPORTED && (attribute.value & VA_RT_FORMAT_YUV420); + } + const std::string & VaapiLibrary::Vendor() const { return vendor; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp new file mode 100644 index 000000000..7d30a0e8a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiVideoDecoderFactory.h" +#include "media/video/codec/linux/VaapiVp9Decoder.h" + +#include "rtc_base/logging.h" + +namespace jni +{ + std::unique_ptr VaapiVideoDecoderFactory::Create() + { + VaapiLibrary * library = VaapiLibrary::Get(); + + if (library == nullptr) { + return nullptr; + } + + const bool vp9 = library->Decodes(VAProfileVP9Profile0); + + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 decoding: " << vp9; + + if (!vp9) { + return nullptr; + } + + return std::unique_ptr(new VaapiVideoDecoderFactory(*library)); + } + + VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library) : + library(library) + { + } + + std::vector VaapiVideoDecoderFactory::GetSupportedFormats() const + { + return { webrtc::SdpVideoFormat::VP9Profile0() }; + } + + std::unique_ptr VaapiVideoDecoderFactory::Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) + { + return std::make_unique(library); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp index d642a144a..47b37bcfa 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp @@ -16,13 +16,18 @@ #include "media/video/codec/linux/VaapiVideoEncoderFactory.h" #include "media/video/codec/linux/VaapiH264Encoder.h" +#include "media/video/codec/linux/VaapiH265Encoder.h" +#include "media/video/codec/linux/VaapiVp9Encoder.h" +#include "media/video/codec/H265Format.h" #include "api/video_codecs/h264_profile_level_id.h" +#include "api/video_codecs/video_codec.h" #include "modules/video_coding/codecs/h264/include/h264.h" +#include "rtc_base/logging.h" namespace jni { - std::unique_ptr VaapiVideoEncoderFactory::Create() + std::unique_ptr VaapiVideoEncoderFactory::Create(bool experimental, bool vp9Enabled) { VaapiLibrary * library = VaapiLibrary::Get(); @@ -30,25 +35,62 @@ namespace jni return nullptr; } - return std::unique_ptr(new VaapiVideoEncoderFactory(*library)); + VAEntrypoint h265Entrypoint = VAEntrypointEncSlice; + const bool h265 = experimental && library->EncodeEntrypoint(VAProfileHEVCMain, &h265Entrypoint); + + VAEntrypoint vp9Entrypoint = VAEntrypointEncSlice; + const bool vp9 = vp9Enabled && library->EncodeEntrypoint(VAProfileVP9Profile0, &vp9Entrypoint); + + if (experimental) { + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.265 encoding: " << h265; + } + if (vp9Enabled) { + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 encoding: " << vp9; + } + + return std::unique_ptr( + new VaapiVideoEncoderFactory(*library, h265, h265Entrypoint, vp9, vp9Entrypoint)); } - VaapiVideoEncoderFactory::VaapiVideoEncoderFactory(VaapiLibrary & library) : - library(library) + VaapiVideoEncoderFactory::VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, + bool vp9, VAEntrypoint vp9Entrypoint) : + library(library), + h265(h265), + h265Entrypoint(h265Entrypoint), + vp9(vp9), + vp9Entrypoint(vp9Entrypoint) { } std::vector VaapiVideoEncoderFactory::GetSupportedFormats() const { - return { + std::vector formats = { webrtc::CreateH264Format(webrtc::H264Profile::kProfileConstrainedBaseline, webrtc::H264Level::kLevel3_1, "1"), webrtc::CreateH264Format(webrtc::H264Profile::kProfileBaseline, webrtc::H264Level::kLevel3_1, "1") }; + + if (h265) { + formats.push_back(CreateH265MainFormat()); + } + if (vp9) { + formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); + } + + return formats; } std::unique_ptr VaapiVideoEncoderFactory::Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) { + const webrtc::VideoCodecType codec = webrtc::PayloadStringToCodecType(format.name); + + if (codec == webrtc::kVideoCodecH265) { + return std::make_unique(library, h265Entrypoint, format); + } + if (codec == webrtc::kVideoCodecVP9) { + return std::make_unique(library, vp9Entrypoint, format); + } + return std::make_unique(library, format); } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Decoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Decoder.cpp new file mode 100644 index 000000000..f1ad72cfd --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Decoder.cpp @@ -0,0 +1,479 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiVp9Decoder.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +#include + +namespace jni +{ + namespace + { + // The eight reference frames, the frame being decoded, and one more, + // so that a free surface is always there. + constexpr int kSurfaceCount = 10; + + // The frame type of an inter frame; a key frame is 0. + constexpr uint32_t kInterFrame = 1; + } + + VaapiVp9Decoder::VaapiVp9Decoder(VaapiLibrary & library) : + library(library), + va(library.Api()), + implementationName("VA-API (" + library.Vendor() + ")"), + callback(nullptr), + config(VA_INVALID_ID), + context(VA_INVALID_ID), + width(0), + height(0), + downloadImage(), + started(false), + failed(false) + { + downloadImage.image_id = VA_INVALID_ID; + slots.fill(-1); + } + + VaapiVp9Decoder::~VaapiVp9Decoder() + { + Release(); + } + + bool VaapiVp9Decoder::Check(VAStatus status, const char * operation) const + { + if (status == VA_STATUS_SUCCESS) { + return true; + } + + RTC_LOG(LS_WARNING) << implementationName << " VP9: " << operation << " failed: " + << library.ErrorString(status); + + return false; + } + + bool VaapiVp9Decoder::Configure(const Settings & settings) + { + if (settings.codec_type() != webrtc::kVideoCodecVP9) { + return false; + } + + Release(); + + RTC_LOG(LS_INFO) << implementationName << " VP9 decoder configured"; + + return true; + } + + int32_t VaapiVp9Decoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + if (callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (image.size() == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // The layers of a VP9 frame with spatial layers reach a decoder back to + // back, without the superframe index that tells where one ends. + if (image.SpatialIndex().value_or(0) > 0 || image.SpatialLayerFrameSize(1).has_value()) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Decoding starts with a key frame. Until one has come, WebRTC is + // asked for it. + if (!started && image.FrameType() != webrtc::VideoFrameType::kVideoFrameKey) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + const auto frames = Vp9FrameHeaderParser::SplitSuperframe(std::span(image.data(), image.size())); + + if (frames.empty()) { + RTC_LOG(LS_WARNING) << implementationName << " VP9: a superframe with a broken index"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + for (std::span frame : frames) { + const std::optional header = parser.Parse(frame); + + if (!header) { + RTC_LOG(LS_INFO) << implementationName << " VP9: a frame that is not 8 bit 4:2:0 profile 0, " + "or refers to a frame that is not there"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + if (header->showExistingFrame) { + if (slots[header->frameToShow] < 0 || !Output(slots[header->frameToShow], image, renderTimeMs)) { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + continue; + } + + const bool sizeChanged = header->width != width || header->height != height; + + if (header->keyFrame && (context == VA_INVALID_ID || sizeChanged)) { + if (!CreateSession(header->width, header->height)) { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + } + else if (context == VA_INVALID_ID || sizeChanged) { + // Inter frames that scale their references are left to libvpx. + RTC_LOG(LS_INFO) << implementationName << " VP9: an inter frame changes the size to " + << header->width << "x" << header->height; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + const int surface = FreeSurface(); + + if (surface < 0 || !DecodeFrame(*header, frame, surface)) { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + for (int slot = 0; slot < 8; slot++) { + if (header->refreshFrameFlags & (1 << slot)) { + slots[slot] = surface; + } + } + + started = true; + + if (header->showFrame && !Output(surface, image, renderTimeMs)) { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiVp9Decoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiVp9Decoder::Release() + { + DestroySession(); + + parser.Reset(); + started = false; + failed = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + webrtc::VideoDecoder::DecoderInfo VaapiVp9Decoder::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + + return info; + } + + const char * VaapiVp9Decoder::ImplementationName() const + { + return implementationName.c_str(); + } + + bool VaapiVp9Decoder::CreateSession(uint32_t frameWidth, uint32_t frameHeight) + { + DestroySession(); + + VADisplay display = library.Display(); + + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribRTFormat; + attribute.value = VA_RT_FORMAT_YUV420; + + if (!Check(va.CreateConfig(display, VAProfileVP9Profile0, VAEntrypointVLD, &attribute, 1, &config), + "creating the configuration")) + { + config = VA_INVALID_ID; + return false; + } + + surfaces.assign(kSurfaceCount, VA_INVALID_SURFACE); + + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, frameWidth, frameHeight, surfaces.data(), + kSurfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); + DestroySession(); + return false; + } + + if (!Check(va.CreateContext(display, config, static_cast(frameWidth), static_cast(frameHeight), + VA_PROGRESSIVE, surfaces.data(), kSurfaceCount, &context), "creating the context")) + { + context = VA_INVALID_ID; + DestroySession(); + return false; + } + + // Copying a surface into an image reads it from cached memory, which + // mapping it directly does not. Where the driver does not copy, the + // surface is mapped. + VAImageFormat format = {}; + format.fourcc = VA_FOURCC_NV12; + format.byte_order = VA_LSB_FIRST; + format.bits_per_pixel = 12; + + if (va.CreateImage(display, &format, static_cast(frameWidth), static_cast(frameHeight), + &downloadImage) != VA_STATUS_SUCCESS) + { + downloadImage.image_id = VA_INVALID_ID; + } + + width = frameWidth; + height = frameHeight; + + RTC_LOG(LS_INFO) << implementationName << " VP9 decoding " << width << "x" << height; + + return true; + } + + void VaapiVp9Decoder::DestroySession() + { + VADisplay display = library.Display(); + + if (downloadImage.image_id != VA_INVALID_ID) { + va.DestroyImage(display, downloadImage.image_id); + downloadImage.image_id = VA_INVALID_ID; + } + if (context != VA_INVALID_ID) { + va.DestroyContext(display, context); + context = VA_INVALID_ID; + } + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); + } + if (config != VA_INVALID_ID) { + va.DestroyConfig(display, config); + config = VA_INVALID_ID; + } + + width = 0; + height = 0; + slots.fill(-1); + } + + int VaapiVp9Decoder::FreeSurface() const + { + for (int surface = 0; surface < static_cast(surfaces.size()); surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + return surface; + } + } + + return -1; + } + + bool VaapiVp9Decoder::DecodeFrame(const Vp9FrameHeader & header, std::span frame, int surface) + { + if (header.uncompressedHeaderSize > UINT8_MAX) { + return false; + } + + VADecPictureParameterBufferVP9 picture = {}; + picture.frame_width = static_cast(header.width); + picture.frame_height = static_cast(header.height); + + for (int slot = 0; slot < 8; slot++) { + picture.reference_frames[slot] = slots[slot] >= 0 ? surfaces[slots[slot]] : VA_INVALID_SURFACE; + } + + auto & fields = picture.pic_fields.bits; + fields.subsampling_x = header.subsamplingX ? 1 : 0; + fields.subsampling_y = header.subsamplingY ? 1 : 0; + fields.frame_type = header.keyFrame ? 0 : kInterFrame; + fields.show_frame = header.showFrame ? 1 : 0; + fields.error_resilient_mode = header.errorResilient ? 1 : 0; + fields.intra_only = header.intraOnly ? 1 : 0; + fields.allow_high_precision_mv = header.allowHighPrecisionMv ? 1 : 0; + fields.mcomp_filter_type = header.interpolationFilter; + fields.frame_parallel_decoding_mode = header.frameParallelDecoding ? 1 : 0; + fields.reset_frame_context = header.resetFrameContext; + fields.refresh_frame_context = header.refreshFrameContext ? 1 : 0; + fields.frame_context_idx = header.frameContextIndex; + fields.segmentation_enabled = header.segmentationEnabled ? 1 : 0; + fields.segmentation_temporal_update = header.segmentationTemporalUpdate ? 1 : 0; + fields.segmentation_update_map = header.segmentationUpdateMap ? 1 : 0; + fields.last_ref_frame = header.referenceSlots[0]; + fields.last_ref_frame_sign_bias = header.referenceSignBias[0] ? 1 : 0; + fields.golden_ref_frame = header.referenceSlots[1]; + fields.golden_ref_frame_sign_bias = header.referenceSignBias[1] ? 1 : 0; + fields.alt_ref_frame = header.referenceSlots[2]; + fields.alt_ref_frame_sign_bias = header.referenceSignBias[2] ? 1 : 0; + fields.lossless_flag = header.lossless ? 1 : 0; + + picture.filter_level = header.filterLevel; + picture.sharpness_level = header.sharpness; + picture.log2_tile_rows = header.log2TileRows; + picture.log2_tile_columns = header.log2TileColumns; + picture.frame_header_length_in_bytes = static_cast(header.uncompressedHeaderSize); + picture.first_partition_size = static_cast(header.compressedHeaderSize); + std::copy(header.segmentationTreeProbs.begin(), header.segmentationTreeProbs.end(), + picture.mb_segment_tree_probs); + std::copy(header.segmentationPredProbs.begin(), header.segmentationPredProbs.end(), + picture.segment_pred_probs); + picture.profile = header.profile; + picture.bit_depth = header.bitDepth; + + VASliceParameterBufferVP9 slice = {}; + slice.slice_data_size = static_cast(frame.size()); + slice.slice_data_offset = 0; + slice.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; + + for (int i = 0; i < 8; i++) { + const Vp9Segment & segment = header.segments[i]; + VASegmentParameterVP9 & target = slice.seg_param[i]; + + target.segment_flags.fields.segment_reference_enabled = segment.referenceEnabled ? 1 : 0; + target.segment_flags.fields.segment_reference = segment.reference; + target.segment_flags.fields.segment_reference_skipped = segment.skipped ? 1 : 0; + + for (int reference = 0; reference < 4; reference++) { + target.filter_level[reference][0] = segment.filterLevel[reference][0]; + target.filter_level[reference][1] = segment.filterLevel[reference][1]; + } + + target.luma_dc_quant_scale = segment.lumaDcScale; + target.luma_ac_quant_scale = segment.lumaAcScale; + target.chroma_dc_quant_scale = segment.chromaDcScale; + target.chroma_ac_quant_scale = segment.chromaAcScale; + } + + VADisplay display = library.Display(); + VABufferID buffers[3] = { VA_INVALID_ID, VA_INVALID_ID, VA_INVALID_ID }; + + bool created = Check(va.CreateBuffer(display, context, VAPictureParameterBufferType, sizeof(picture), 1, + &picture, &buffers[0]), "creating the picture parameters") + && Check(va.CreateBuffer(display, context, VASliceParameterBufferType, sizeof(slice), 1, &slice, + &buffers[1]), "creating the slice parameters") + && Check(va.CreateBuffer(display, context, VASliceDataBufferType, static_cast(frame.size()), + 1, const_cast(frame.data()), &buffers[2]), "creating the slice data"); + + bool decoded = false; + + if (created && Check(va.BeginPicture(display, context, surfaces[surface]), "beginning a picture")) { + const bool rendered = Check(va.RenderPicture(display, context, buffers, 3), "rendering a picture"); + // A picture that was begun has to be ended. + const bool ended = Check(va.EndPicture(display, context), "ending a picture"); + + decoded = rendered && ended && Check(va.SyncSurface(display, surfaces[surface]), "waiting for a picture"); + } + + for (VABufferID buffer : buffers) { + if (buffer != VA_INVALID_ID) { + va.DestroyBuffer(display, buffer); + } + } + + return decoded; + } + + bool VaapiVp9Decoder::Output(int surface, const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + VADisplay display = library.Display(); + const int frameWidth = static_cast(width); + const int frameHeight = static_cast(height); + + VAImage mapped = {}; + mapped.image_id = VA_INVALID_ID; + bool derived = false; + + if (downloadImage.image_id != VA_INVALID_ID + && va.GetImage(display, surfaces[surface], 0, 0, width, height, downloadImage.image_id) + == VA_STATUS_SUCCESS) + { + mapped = downloadImage; + } + else if (va.DeriveImage(display, surfaces[surface], &mapped) == VA_STATUS_SUCCESS) { + derived = true; + + if (mapped.format.fourcc != VA_FOURCC_NV12) { + RTC_LOG(LS_WARNING) << implementationName << " VP9: the decoded surface is not NV12"; + va.DestroyImage(display, mapped.image_id); + return false; + } + } + else { + RTC_LOG(LS_WARNING) << implementationName << " VP9: the decoded surface cannot be read"; + return false; + } + + void * data = nullptr; + bool converted = false; + webrtc::scoped_refptr i420 = webrtc::I420Buffer::Create(frameWidth, frameHeight); + + if (Check(va.MapBuffer(display, mapped.buf, &data), "mapping a decoded picture")) { + const uint8_t * base = static_cast(data); + + converted = libyuv::NV12ToI420(base + mapped.offsets[0], static_cast(mapped.pitches[0]), + base + mapped.offsets[1], static_cast(mapped.pitches[1]), + i420->MutableDataY(), i420->StrideY(), i420->MutableDataU(), i420->StrideU(), + i420->MutableDataV(), i420->StrideV(), frameWidth, frameHeight) == 0; + + va.UnmapBuffer(display, mapped.buf); + } + + if (derived) { + va.DestroyImage(display, mapped.image_id); + } + + if (!converted) { + return false; + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420) + .set_rtp_timestamp(image.RtpTimestamp()) + .set_timestamp_ms(renderTimeMs) + .set_ntp_time_ms(image.ntp_time_ms_) + .set_rotation(image.rotation_) + .build(); + + callback->Decoded(frame, std::nullopt, std::nullopt); + + return true; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp new file mode 100644 index 000000000..139a59afa --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -0,0 +1,287 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiVp9Encoder.h" + +#include "api/video/encoded_image.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +#include + +namespace jni +{ + namespace + { + // Key frames come when WebRTC asks for them; this only bounds how + // long a stream may go without one if it never does. + constexpr uint32_t kKeyFrameInterval = 3000; + + // Surfaces are kept to whole superblocks, which the driver may write + // all of, as FFmpeg's VA-API encoder does. + constexpr uint32_t kSurfaceAlignment = 64; + + // VP9 frame types (7.2 of the bitstream specification). + constexpr uint32_t kKeyFrameType = 0; + constexpr uint32_t kInterFrameType = 1; + + // The quantizer index and loop filter settings the driver starts + // from, FFmpeg's defaults; the rate control sets the quantizer of each + // frame. + constexpr uint8_t kQIndex = 100; + constexpr uint8_t kFilterLevel = 16; + constexpr uint8_t kSharpnessLevel = 4; + + // The reference slot the next frame refers to as its last frame. + constexpr uint32_t kLastSlot = 0; + + uint32_t Align(uint32_t value, uint32_t alignment) + { + return (value + alignment - 1) / alignment * alignment; + } + } + + VaapiVp9Encoder::VaapiVp9Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, + const webrtc::SdpVideoFormat & format) : + library(library), + entrypoint(entrypoint), + implementationName("VA-API (" + library.Vendor() + ")"), + session(library, "VA-API VP9"), + initialized(false), + codecSettings(), + bitrateBps(0), + framerate(30), + ratesChanged(false), + current(0), + referenceValid(false), + callback(nullptr), + outputProcessor(webrtc::kVideoCodecVP9, format) + { + } + + VaapiVp9Encoder::~VaapiVp9Encoder() + { + Release(); + } + + int32_t VaapiVp9Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + if (settings == nullptr || settings->width == 0 || settings->height == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (settings->numberOfSimulcastStreams > 1) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + // One spatial and one temporal layer is all this encoder produces. + if (settings->VP9().numberOfSpatialLayers > 1 || settings->VP9().numberOfTemporalLayers > 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + Release(); + + codecSettings = *settings; + bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; + framerate = std::max(1u, codecSettings.maxFramerate); + ratesChanged = true; + current = 0; + referenceValid = false; + outputProcessor.Reset(); + + if (!session.Create(VAProfileVP9Profile0, entrypoint, Align(codecSettings.width, kSurfaceAlignment), + Align(codecSettings.height, kSurfaceAlignment))) + { + Release(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + initialized = true; + + RTC_LOG(LS_INFO) << implementationName << " VP9 initialized: " << codecSettings.width << "x" + << codecSettings.height << " at " << bitrateBps << " bps"; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiVp9Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiVp9Encoder::Release() + { + session.Destroy(); + + initialized = false; + referenceValid = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiVp9Encoder::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized || callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + bool keyFrame = !referenceValid; + + if (frameTypes != nullptr) { + keyFrame |= std::any_of(frameTypes->begin(), frameTypes->end(), [](webrtc::VideoFrameType type) { + return type == webrtc::VideoFrameType::kVideoFrameKey; + }); + } + + std::vector output; + + // VP9 codes any size, so the frame is taken as it is, and scaled only + // if it raced with a change of size. + const bool encoded = session.Upload(frame, static_cast(codecSettings.width), + static_cast(codecSettings.height), 0) && Submit(keyFrame) && session.ReadOutput(output); + + session.DropBuffers(); + + if (!encoded) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + webrtc::EncodedImage image; + image._encodedWidth = codecSettings.width; + image._encodedHeight = codecSettings.height; + image.SetRtpTimestamp(frame.rtp_timestamp()); + image.capture_time_ms_ = frame.render_time_ms(); + image.ntp_time_ms_ = frame.ntp_time_ms(); + image.rotation_ = frame.rotation(); + + webrtc::CodecSpecificInfo info; + + if (!outputProcessor.Process(output, keyFrame, image, info)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // The frame just encoded is the reference of the next one. + referenceValid = true; + current = 1 - current; + ratesChanged = false; + + callback->OnEncodedImage(image, &info); + + return WEBRTC_VIDEO_CODEC_OK; + } + + bool VaapiVp9Encoder::Submit(bool keyFrame) + { + if (keyFrame) { + VAEncSequenceParameterBufferVP9 sequence = {}; + FillSequence(sequence); + + if (!session.AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { + return false; + } + } + + // No initial quantizer: VP9 rate control picks its own, as FFmpeg + // leaves it. + if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + return false; + } + + VAEncPictureParameterBufferVP9 picture = {}; + FillPicture(picture, keyFrame); + + return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) && session.Encode(); + } + + void VaapiVp9Encoder::FillSequence(VAEncSequenceParameterBufferVP9 & sequence) const + { + sequence.max_frame_width = codecSettings.width; + sequence.max_frame_height = codecSettings.height; + // Key frames only where asked for. + sequence.kf_auto = 0; + sequence.kf_min_dist = 1; + sequence.kf_max_dist = kKeyFrameInterval; + sequence.intra_period = kKeyFrameInterval; + sequence.bits_per_second = bitrateBps; + } + + void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const + { + picture.frame_width_src = codecSettings.width; + picture.frame_height_src = codecSettings.height; + picture.frame_width_dst = codecSettings.width; + picture.frame_height_dst = codecSettings.height; + picture.reconstructed_frame = session.Reconstructed(current); + picture.coded_buf = session.CodedBuffer(); + + for (VASurfaceID & reference : picture.reference_frames) { + reference = VA_INVALID_SURFACE; + } + + if (keyFrame) { + picture.ref_flags.bits.force_kf = 1; + picture.pic_flags.bits.frame_type = kKeyFrameType; + } + else { + picture.reference_frames[kLastSlot] = session.Reconstructed(1 - current); + // The last frame is the only reference, in list 0. + picture.ref_flags.bits.ref_frame_ctrl_l0 = 1; + picture.ref_flags.bits.ref_last_idx = kLastSlot; + picture.pic_flags.bits.frame_type = kInterFrameType; + } + + picture.pic_flags.bits.show_frame = 1; + // Every slot holds this frame afterwards. + picture.refresh_frame_flags = 0xFF; + + picture.luma_ac_qindex = kQIndex; + picture.filter_level = kFilterLevel; + picture.sharpness_level = kSharpnessLevel; + } + + void VaapiVp9Encoder::SetRates(const RateControlParameters & parameters) + { + const uint32_t bitrate = parameters.bitrate.get_sum_bps(); + + // A zero bitrate pauses the stream; the encoder gets no frames then. + if (bitrate == 0) { + return; + } + + bitrateBps = bitrate; + + if (parameters.framerate_fps >= 1.0) { + framerate = static_cast(parameters.framerate_fps + 0.5); + } + + // Applied with the next frame. + ratesChanged = true; + } + + webrtc::VideoEncoder::EncoderInfo VaapiVp9Encoder::GetEncoderInfo() const + { + EncoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + info.supports_native_handle = false; + info.supports_simulcast = false; + info.scaling_settings = outputProcessor.GetScalingSettings(); + + return info; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Vp9FrameHeaderParser.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Vp9FrameHeaderParser.cpp new file mode 100644 index 000000000..9346ea3d5 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Vp9FrameHeaderParser.cpp @@ -0,0 +1,462 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/Vp9FrameHeaderParser.h" + +#include "rtc_base/bitstream_reader.h" + +#include + +// The quantizer lookup of libvpx, which WebRTC links for its VP9 codecs. +// Declared here since its headers are not among WebRTC's; the bit depth is +// an enum there, 8 for 8 bit. +extern "C" { + int16_t vp9_dc_quant(int qindex, int delta, int bitDepth); + int16_t vp9_ac_quant(int qindex, int delta, int bitDepth); +} + +namespace jni +{ + namespace + { + constexpr uint32_t kFrameSyncCode = 0x498342; + constexpr uint8_t kColorSpaceRgb = 7; + + constexpr int kMaxLoopFilter = 63; + constexpr int kMaxQIndex = 255; + + // The segmentation features: alternate quantizer, alternate loop + // filter level, reference frame and skip. + constexpr int kFeatureQuantizer = 0; + constexpr int kFeatureLoopFilter = 1; + constexpr int kFeatureReference = 2; + constexpr int kFeatureSkip = 3; + constexpr int kFeatureBits[4] = { 8, 6, 2, 0 }; + constexpr bool kFeatureSigned[4] = { true, true, false, false }; + + // Tiles are 4 to 64 superblocks of 64x64 wide. + constexpr uint32_t kMinTileWidthB64 = 4; + constexpr uint32_t kMaxTileWidthB64 = 64; + + // su(n) of the specification: a magnitude of n bits and a sign. + int ReadSigned(webrtc::BitstreamReader & reader, int bits) + { + const int value = static_cast(reader.ReadBits(bits)); + + return reader.Read() ? -value : value; + } + + uint8_t ReadProbability(webrtc::BitstreamReader & reader) + { + return reader.Read() ? reader.Read() : 255; + } + + int8_t ReadDeltaQ(webrtc::BitstreamReader & reader) + { + return reader.Read() ? static_cast(ReadSigned(reader, 4)) : 0; + } + + uint8_t ClampLevel(int level) + { + return static_cast(std::clamp(level, 0, kMaxLoopFilter)); + } + } + + Vp9FrameHeaderParser::Vp9FrameHeaderParser() + { + Reset(); + } + + void Vp9FrameHeaderParser::Reset() + { + SetupPastIndependence(); + + segmentationTreeProbs.fill(255); + segmentationPredProbs.fill(255); + slotWidths.fill(0); + slotHeights.fill(0); + } + + void Vp9FrameHeaderParser::SetupPastIndependence() + { + loopFilterDeltaEnabled = true; + loopFilterRefDeltas = { 1, 0, -1, -1 }; + loopFilterModeDeltas = { 0, 0 }; + + segmentationAbsoluteValues = false; + + for (int segment = 0; segment < 8; segment++) { + for (int feature = 0; feature < 4; feature++) { + featureEnabled[segment][feature] = false; + featureData[segment][feature] = 0; + } + } + } + + std::optional Vp9FrameHeaderParser::Parse(std::span frame) + { + webrtc::BitstreamReader reader(frame); + Vp9FrameHeader header; + + if (reader.ReadBits(2) != 2) { + return std::nullopt; + } + + // The profile has its low bit first. + header.profile = static_cast(reader.ReadBit()); + header.profile |= static_cast(reader.ReadBit() << 1); + + if (header.profile != 0) { + return std::nullopt; + } + + if (reader.Read()) { + header.showExistingFrame = true; + header.frameToShow = static_cast(reader.ReadBits(3)); + + if (!reader.Ok() || slotWidths[header.frameToShow] == 0) { + return std::nullopt; + } + + header.width = slotWidths[header.frameToShow]; + header.height = slotHeights[header.frameToShow]; + + return header; + } + + header.keyFrame = !reader.Read(); + header.showFrame = reader.Read(); + header.errorResilient = reader.Read(); + + bool readSize = true; + + if (header.keyFrame) { + if (reader.ReadBits(24) != kFrameSyncCode) { + return std::nullopt; + } + + // The color configuration of profile 0: 8 bit 4:2:0, whose + // color space must not be RGB, which needs 4:4:4. + if (reader.ReadBits(3) == kColorSpaceRgb) { + return std::nullopt; + } + + reader.ConsumeBits(1); + + header.refreshFrameFlags = 0xFF; + } + else { + header.intraOnly = header.showFrame ? false : reader.Read(); + header.resetFrameContext = header.errorResilient ? 0 : static_cast(reader.ReadBits(2)); + + if (header.intraOnly) { + if (reader.ReadBits(24) != kFrameSyncCode) { + return std::nullopt; + } + + // Profile 0 carries no color configuration here: 8 bit 4:2:0. + header.refreshFrameFlags = reader.Read(); + } + else { + header.refreshFrameFlags = reader.Read(); + + for (int i = 0; i < 3; i++) { + header.referenceSlots[i] = static_cast(reader.ReadBits(3)); + header.referenceSignBias[i] = reader.Read(); + + if (slotWidths[header.referenceSlots[i]] == 0) { + return std::nullopt; + } + } + + // The size of a reference frame, or one of its own. + for (int i = 0; i < 3; i++) { + if (reader.Read()) { + header.width = slotWidths[header.referenceSlots[i]]; + header.height = slotHeights[header.referenceSlots[i]]; + readSize = false; + break; + } + } + } + } + + if (readSize) { + header.width = static_cast(reader.Read()) + 1; + header.height = static_cast(reader.Read()) + 1; + } + + // The render size, which a decoder does not need. + if (reader.Read()) { + reader.ConsumeBits(32); + } + + if (!header.keyFrame && !header.intraOnly) { + header.allowHighPrecisionMv = reader.Read(); + header.interpolationFilter = reader.Read() ? 4 : static_cast(reader.ReadBits(2)); + } + + if (!header.errorResilient) { + header.refreshFrameContext = reader.Read(); + header.frameParallelDecoding = reader.Read(); + } + else { + header.refreshFrameContext = false; + header.frameParallelDecoding = true; + } + + header.frameContextIndex = static_cast(reader.ReadBits(2)); + + if (header.keyFrame || header.intraOnly || header.errorResilient) { + SetupPastIndependence(); + } + // As FFmpeg does, which drivers are used with: the index of a key + // frame or an intra-only frame is not used. + if (header.keyFrame || header.intraOnly) { + header.frameContextIndex = 0; + } + + // Loop filter parameters. + header.filterLevel = static_cast(reader.ReadBits(6)); + header.sharpness = static_cast(reader.ReadBits(3)); + loopFilterDeltaEnabled = reader.Read(); + + if (loopFilterDeltaEnabled && reader.Read()) { + for (int8_t & delta : loopFilterRefDeltas) { + if (reader.Read()) { + delta = static_cast(ReadSigned(reader, 6)); + } + } + for (int8_t & delta : loopFilterModeDeltas) { + if (reader.Read()) { + delta = static_cast(ReadSigned(reader, 6)); + } + } + } + + // Quantization parameters. + header.baseQIndex = reader.Read(); + const int8_t yDcDelta = ReadDeltaQ(reader); + const int8_t uvDcDelta = ReadDeltaQ(reader); + const int8_t uvAcDelta = ReadDeltaQ(reader); + header.lossless = header.baseQIndex == 0 && yDcDelta == 0 && uvDcDelta == 0 && uvAcDelta == 0; + + // Segmentation parameters. + header.segmentationEnabled = reader.Read(); + + if (header.segmentationEnabled) { + header.segmentationUpdateMap = reader.Read(); + + if (header.segmentationUpdateMap) { + for (uint8_t & probability : segmentationTreeProbs) { + probability = ReadProbability(reader); + } + + header.segmentationTemporalUpdate = reader.Read(); + + for (uint8_t & probability : segmentationPredProbs) { + probability = header.segmentationTemporalUpdate ? ReadProbability(reader) : 255; + } + } + + if (reader.Read()) { + segmentationAbsoluteValues = reader.Read(); + + for (int segment = 0; segment < 8; segment++) { + for (int feature = 0; feature < 4; feature++) { + int value = 0; + + featureEnabled[segment][feature] = reader.Read(); + + if (featureEnabled[segment][feature]) { + value = static_cast(reader.ReadBits(kFeatureBits[feature])); + + if (kFeatureSigned[feature] && reader.Read()) { + value = -value; + } + } + + featureData[segment][feature] = static_cast(value); + } + } + } + } + + header.segmentationTreeProbs = segmentationTreeProbs; + header.segmentationPredProbs = segmentationPredProbs; + + // Tile information. + const uint32_t miColumns = (header.width + 7) >> 3; + const uint32_t sb64Columns = (miColumns + 7) >> 3; + + uint8_t minLog2 = 0; + + while ((kMaxTileWidthB64 << minLog2) < sb64Columns) { + minLog2++; + } + + uint8_t maxLog2 = 1; + + while ((sb64Columns >> maxLog2) >= kMinTileWidthB64) { + maxLog2++; + } + + maxLog2--; + + header.log2TileColumns = minLog2; + + while (header.log2TileColumns < maxLog2 && reader.Read()) { + header.log2TileColumns++; + } + + header.log2TileRows = reader.Read() ? (reader.Read() ? 2 : 1) : 0; + + header.compressedHeaderSize = reader.Read(); + + if (!reader.Ok() || header.compressedHeaderSize == 0) { + return std::nullopt; + } + + // The uncompressed header ends on a whole byte. + const size_t bitsRead = frame.size() * 8 - static_cast(reader.RemainingBitCount()); + header.uncompressedHeaderSize = static_cast((bitsRead + 7) / 8); + + if (header.uncompressedHeaderSize + header.compressedHeaderSize > frame.size()) { + return std::nullopt; + } + + ComputeSegments(header, yDcDelta, uvDcDelta, uvAcDelta); + + for (int slot = 0; slot < 8; slot++) { + if (header.refreshFrameFlags & (1 << slot)) { + slotWidths[slot] = header.width; + slotHeights[slot] = header.height; + } + } + + return header; + } + + void Vp9FrameHeaderParser::ComputeSegments(Vp9FrameHeader & header, int8_t yDcDelta, int8_t uvDcDelta, + int8_t uvAcDelta) const + { + // The loop filter deltas count double from level 32 on. + const int scale = 1 << (header.filterLevel >> 5); + + for (int segment = 0; segment < 8; segment++) { + Vp9Segment & target = header.segments[segment]; + + auto active = [&](int feature) { + return header.segmentationEnabled && featureEnabled[segment][feature]; + }; + + target.referenceEnabled = active(kFeatureReference); + target.reference = static_cast(featureData[segment][kFeatureReference]); + target.skipped = active(kFeatureSkip); + + int qIndex = header.baseQIndex; + + if (active(kFeatureQuantizer)) { + const int data = featureData[segment][kFeatureQuantizer]; + + qIndex = std::clamp(segmentationAbsoluteValues ? data : qIndex + data, 0, kMaxQIndex); + } + + target.lumaDcScale = vp9_dc_quant(qIndex, yDcDelta, 8); + target.lumaAcScale = vp9_ac_quant(qIndex, 0, 8); + target.chromaDcScale = vp9_dc_quant(qIndex, uvDcDelta, 8); + target.chromaAcScale = vp9_ac_quant(qIndex, uvAcDelta, 8); + + int level = header.filterLevel; + + if (active(kFeatureLoopFilter)) { + const int data = featureData[segment][kFeatureLoopFilter]; + + level = ClampLevel(segmentationAbsoluteValues ? data : level + data); + } + + if (!loopFilterDeltaEnabled) { + for (auto & modes : target.filterLevel) { + modes[0] = static_cast(level); + modes[1] = static_cast(level); + } + + continue; + } + + // Intra blocks have no mode delta. + const uint8_t intraLevel = ClampLevel(level + loopFilterRefDeltas[0] * scale); + target.filterLevel[0][0] = intraLevel; + target.filterLevel[0][1] = intraLevel; + + for (int reference = 1; reference < 4; reference++) { + for (int mode = 0; mode < 2; mode++) { + target.filterLevel[reference][mode] = ClampLevel(level + loopFilterRefDeltas[reference] * scale + + loopFilterModeDeltas[mode] * scale); + } + } + } + } + + std::vector> Vp9FrameHeaderParser::SplitSuperframe(std::span data) + { + std::vector> frames; + + if (data.empty()) { + return frames; + } + + // A superframe ends in an index: a marker byte, the sizes of its + // frames, and the marker again. + const uint8_t marker = data.back(); + + if ((marker & 0xE0) == 0xC0) { + const size_t count = (marker & 0x07) + 1; + const size_t sizeBytes = ((marker >> 3) & 0x03) + 1; + const size_t indexSize = 2 + sizeBytes * count; + + if (data.size() >= indexSize && data[data.size() - indexSize] == marker) { + const size_t end = data.size() - indexSize; + size_t entry = end + 1; + size_t offset = 0; + + for (size_t i = 0; i < count; i++) { + size_t size = 0; + + // Little endian. + for (size_t b = 0; b < sizeBytes; b++) { + size |= static_cast(data[entry++]) << (8 * b); + } + + if (size > end - offset) { + return {}; + } + if (size > 0) { + frames.push_back(data.subspan(offset, size)); + } + + offset += size; + } + + return frames; + } + } + + frames.push_back(data); + + return frames; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index bbea9d8e7..4165e1ccc 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -54,7 +54,9 @@ * On Linux, H.264 and VP9 (profile 0) are decoded on NVIDIA GPUs with NVDEC, * loaded from the NVIDIA driver when the factory is made; a machine without * the driver is not affected. Streams with spatial layers go to libvpx. - * Decoding with VA-API, on the GPUs of Intel and AMD, is not there yet. + * VP9 (profile 0) is decoded with VA-API on the GPUs of Intel and AMD only + * with the system property {@code webrtc.hardware.vaapiExperimental} set to + * {@code true}: the decoder has not been tested on such a GPU yet. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. H.265 comes after them, only where the hardware decodes it: diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index f181aef1f..d9990ebd7 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -40,11 +40,15 @@ * have an AV1 encoder, with NVENC or Media Foundation, as a single layer; a * stream that asks for SVC is encoded in software. H.265 (Main profile) is * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. - * VP9 (profile 0) is encoded with Media Foundation on GPUs that have a VP9 - * encoder, which on Windows are Intel's, but only with the system property + * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are + * Intel's, with Media Foundation on Windows and VA-API on Linux, but only + * with the system property * {@code webrtc.hardware.vp9Encoder} set to {@code true} when the factory is * created: the encoder has not been tested on such a GPU yet. Without it, VP9 - * is encoded with libvpx. + * is encoded with libvpx. On Linux, H.265 is encoded with VA-API on the GPUs + * of Intel and AMD only with the system property + * {@code webrtc.hardware.vaapiExperimental} set to {@code true}, for the same + * reason; NVENC encodes it regardless. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one * in line, and finally by the software encoder of the same codec, so a stream diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index a2062b26c..61f9738bc 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -548,7 +548,7 @@ private static void assumeVp9Platform() { private static boolean isHardware(String implementation) { return implementation.contains("VideoToolbox") || implementation.contains("MediaFoundation") - || implementation.contains("NVDEC"); + || implementation.contains("NVDEC") || implementation.contains("VA-API"); } /** diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 48ea15c8e..23ec7a3d7 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -18,25 +18,37 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; import dev.onvoid.webrtc.media.video.VideoTrack; import dev.onvoid.webrtc.media.video.VideoTrackSink; import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.EncodedImage; import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.VideoCodecInfo; +import dev.onvoid.webrtc.media.video.codec.VideoCodecStatus; +import dev.onvoid.webrtc.media.video.codec.VideoDecoder; +import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; +import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import org.junit.jupiter.api.Test; @@ -52,8 +64,8 @@ * {@code webrtc.test.hardwareAv1Encoder} for a GPU that encodes AV1, * {@code webrtc.test.hardwareH265Encoder} for one that encodes and decodes * H.265, which a call in H.265 needs at both ends, having no software codec, - * and {@code webrtc.test.hardwareVp9Encoder} for one that encodes VP9 (Intel, - * on Windows), which is tested with hardware VP9 encoding turned on. + * and {@code webrtc.test.hardwareVp9Encoder} for one that encodes VP9 (Intel), + * which is tested with hardware VP9 encoding turned on. */ @Execution(ExecutionMode.SAME_THREAD) class HardwareVideoEncoderIntegrationTest extends TestBase { @@ -71,6 +83,9 @@ class HardwareVideoEncoderIntegrationTest extends TestBase { /** The property that turns on hardware VP9 encoding, which is off by default. */ private static final String VP9_ENCODER_PROPERTY = "webrtc.hardware.vp9Encoder"; + /** The property that turns on VA-API's codecs beyond H.264, off by default. */ + private static final String VAAPI_EXPERIMENTAL_PROPERTY = "webrtc.hardware.vaapiExperimental"; + private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT); /** @@ -280,7 +295,8 @@ void vp9HardwareEncodingIsOptIn() throws Exception { @Test void hardwareEncodesVp9WhenEnabled() throws Exception { - assumeTrue(OS.contains("win"), "hardware VP9 encoding is implemented on Windows only"); + assumeTrue(OS.contains("win") || OS.contains("linux"), + "hardware VP9 encoding is implemented on Windows and Linux only"); String previous = System.setProperty(VP9_ENCODER_PROPERTY, "true"); @@ -306,6 +322,57 @@ void hardwareEncodesVp9WhenEnabled() throws Exception { } } + @Test + void hardwareH265EncoderSendsWholeAccessUnits() throws Exception { + assumeTrue(OS.contains("win") || OS.contains("linux"), + "hardware encoders are implemented on Windows and Linux only"); + + // On Linux, VA-API's H.265 encoder is there only when turned on. + String previous = System.setProperty(VAAPI_EXPERIMENTAL_PROPERTY, "true"); + + try { + // The receiver needs no H.265 decoder of its own: a Java decoder + // checks what arrives, so that a GPU that encodes but does not + // decode H.265 is tested as well. + boolean available = offersH265(new HardwareVideoEncoderFactory().getSupportedCodecs()); + + if (HARDWARE_H265_REQUIRED) { + assertTrue(available, "no hardware H.265 encoder"); + } + else { + assumeTrue(available, "no hardware H.265 encoder"); + } + + H265CheckingDecoderFactory decoders = new H265CheckingDecoderFactory(); + PeerConnectionFactory hardware = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .setVideoDecoderFactory(decoders) + .build(); + + // 640x360, which hardware H.265 encoders take (see above). + try (Run run = new Run(hardware, H265, 640, 360)) { + run.awaitFrames(10); + + assertHardware(run.implementation(), true); + } + finally { + hardware.dispose(); + } + + assertNull(decoders.problem.get(), decoders.problem.get()); + assertTrue(decoders.keyFrames.get() > 0, "no key frame arrived"); + } + finally { + if (previous == null) { + System.clearProperty(VAAPI_EXPERIMENTAL_PROPERTY); + } + else { + System.setProperty(VAAPI_EXPERIMENTAL_PROPERTY, previous); + } + } + } + private static boolean offersH265(List codecs) { return codecs.stream().anyMatch(codec -> "H265".equalsIgnoreCase(codec.getName())); } @@ -704,4 +771,109 @@ private interface Condition { } } + + + /** + * Receives H.265 without decoding it: checks that every key frame is a + * whole access unit, with its parameter sets (VPS, SPS, PPS) and an IRAP + * picture, and hands back a blank frame of the encoded size for each + * frame, so that the call goes on. Every other codec goes to the default + * decoders. + */ + private static final class H265CheckingDecoderFactory implements VideoDecoderFactory { + + final DefaultVideoDecoderFactory builtIn = new DefaultVideoDecoderFactory(); + final AtomicInteger keyFrames = new AtomicInteger(); + final AtomicReference problem = new AtomicReference<>(); + + + @Override + public List getSupportedCodecs() { + Map parameters = new HashMap<>(); + parameters.put("profile-id", "1"); + parameters.put("tier-flag", "0"); + parameters.put("level-id", "93"); + parameters.put("tx-mode", "SRST"); + + List codecs = new ArrayList<>(builtIn.getSupportedCodecs()); + codecs.add(new VideoCodecInfo("H265", parameters)); + return codecs; + } + + @Override + public VideoDecoder createDecoder(VideoCodecInfo info) { + if (!"H265".equalsIgnoreCase(info.getName())) { + return builtIn.createDecoder(info); + } + + return new VideoDecoder() { + + private Callback callback; + private int width = 640; + private int height = 360; + + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + Set types = nalUnitTypes(image.getBuffer()); + + if (image.getFrameType() == EncodedImage.FrameType.KEY) { + boolean parameterSets = types.contains(32) && types.contains(33) && types.contains(34); + boolean irap = types.stream().anyMatch(type -> type >= 16 && type <= 23); + + if (parameterSets && irap) { + keyFrames.incrementAndGet(); + } + else { + problem.compareAndSet(null, "a key frame with the NAL unit types " + types); + } + } + else if (types.isEmpty()) { + problem.compareAndSet(null, "a frame without NAL units"); + } + + if (image.getEncodedWidth() > 0 && image.getEncodedHeight() > 0) { + width = image.getEncodedWidth(); + height = image.getEncodedHeight(); + } + + VideoFrame frame = new VideoFrame(NativeI420Buffer.allocate(width, height), + image.getCaptureTimeNs()); + callback.onDecodedFrame(frame, null, null); + frame.release(); + + return VideoCodecStatus.OK; + } + }; + } + + /** The types of the NAL units of an access unit in Annex B form. */ + private static Set nalUnitTypes(ByteBuffer accessUnit) { + Set types = new TreeSet<>(); + ByteBuffer data = accessUnit.duplicate(); + int limit = data.limit(); + + for (int i = data.position(); i + 3 < limit; i++) { + if (data.get(i) == 0 && data.get(i + 1) == 0 && data.get(i + 2) == 1) { + types.add((data.get(i + 3) >> 1) & 0x3F); + i += 2; + } + } + + return types; + } + + } + }