From 4d480ec231c3b82030db59a211b6fbb68b0a1719 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 18:36:32 +0200 Subject: [PATCH 1/4] feat: list the hardware video codecs in CodecListExample The example builds its PeerConnectionFactory with the hardware video codec factories, so the RTP capabilities include the codecs only the GPU handles, and compares the codecs of the default and the hardware factories, marking those only available in hardware. --- docs/guide/examples.md | 3 +- .../webrtc/examples/CodecListExample.java | 92 ++++++++++++++++++- 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 5e9a4e9a7..31fa27d2f 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -32,9 +32,10 @@ This example is useful for applications that need to receive media streams from The [`CodecListExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java) demonstrates how to list all supported codecs with the WebRTC peer-connection-factory. **Key features demonstrated:** -- Creating a `PeerConnectionFactory` +- Creating a `PeerConnectionFactory` with the `HardwareVideoEncoderFactory` and `HardwareVideoDecoderFactory` - Getting the supported codecs for both sending and receiving audio and video - Displaying detailed information about each codec +- Comparing the codecs of the default and the hardware video codec factories, and marking those only available in hardware (H.265) This example is useful for understanding what codecs are available on the current system, which can help with debugging compatibility issues or optimizing media quality. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java index 872255119..55d3665ff 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java @@ -23,25 +23,54 @@ import dev.onvoid.webrtc.RTCRtpCapabilities; import dev.onvoid.webrtc.RTCRtpCodecCapability; import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoCodecInfo; /** * Example demonstrating how to list all supported codecs with the WebRTC peer-connection-factory. *

* This example shows how to: *

- * + *

+ * The hardware factories support the codecs of the default factories, in the same order, and add + * those the GPU handles that have no software implementation (H.265). Whether a codec that both + * support is encoded or decoded on the GPU is decided per stream, and shows in the stats of a + * running connection ({@code powerEfficientEncoder} and {@code powerEfficientDecoder}). + * * @author Alex Andres */ public class CodecListExample { public static void main(String[] args) { - PeerConnectionFactory factory = new PeerConnectionFactory(); + HardwareVideoEncoderFactory hardwareEncoderFactory = new HardwareVideoEncoderFactory(); + HardwareVideoDecoderFactory hardwareDecoderFactory = new HardwareVideoDecoderFactory(); + + PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setVideoEncoderFactory(hardwareEncoderFactory) + .setVideoDecoderFactory(hardwareDecoderFactory) + .build(); try { + // Compare the video codecs of the default and the hardware factories. + System.out.println("\nVIDEO CODEC FACTORIES:"); + System.out.println("---------------------"); + + System.out.println("\nVideo Encoders:"); + printFactoryCodecs(new DefaultVideoEncoderFactory().getSupportedCodecs(), + hardwareEncoderFactory.getSupportedCodecs()); + + System.out.println("\nVideo Decoders:"); + printFactoryCodecs(new DefaultVideoDecoderFactory().getSupportedCodecs(), + hardwareDecoderFactory.getSupportedCodecs()); + // Get receiver capabilities for audio and video. System.out.println("\nRECEIVER CAPABILITIES:"); System.out.println("---------------------"); @@ -72,6 +101,63 @@ public static void main(String[] args) { } } + /** + * Prints the codecs of a hardware video codec factory, marking those the default factory does + * not have, which are only available in hardware. + * + * @param defaultCodecs The codecs of the default factory. + * @param hardwareCodecs The codecs of the hardware factory. + */ + private static void printFactoryCodecs(List defaultCodecs, + List hardwareCodecs) { + if (hardwareCodecs.isEmpty()) { + System.out.println(" No codecs found."); + return; + } + + for (int i = 0; i < hardwareCodecs.size(); i++) { + VideoCodecInfo codec = hardwareCodecs.get(i); + StringBuilder sb = new StringBuilder(); + + sb.append(" Codec #").append(i + 1).append(": "); + sb.append("Name: ").append(codec.getName()); + + Map parameters = codec.getParameters(); + if (!parameters.isEmpty()) { + sb.append(" | Params: ").append(parameters); + } + + List scalabilityModes = codec.getScalabilityModes(); + if (!scalabilityModes.isEmpty()) { + sb.append(" | Scalability Modes: ").append(scalabilityModes.size()); + } + + if (!containsCodec(defaultCodecs, codec)) { + sb.append(" | HARDWARE ONLY"); + } + + System.out.println(sb); + } + } + + /** + * Checks whether the list holds a codec with the same name and format parameters. + * + * @param codecs The codecs to search. + * @param codec The codec to look for. + * + * @return true if the list holds the codec. + */ + private static boolean containsCodec(List codecs, VideoCodecInfo codec) { + for (VideoCodecInfo other : codecs) { + if (other.getName().equalsIgnoreCase(codec.getName()) + && other.getParameters().equals(codec.getParameters())) { + return true; + } + } + return false; + } + /** * Prints detailed information about each codec in the provided list. * All information for a codec is printed in a single line for conciseness. From f15391698029950094b42ca77fe2a61c72a88ef5 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 21:33:41 +0200 Subject: [PATCH 2/4] feat: list the hardware codecs of the hardware video codec factories HardwareVideoEncoderFactory and HardwareVideoDecoderFactory get getHardwareCodecs(): the supported codecs, in their order, that a hardware encoder or decoder of the machine takes first, and on macOS H.264, which the default factories handle with VideoToolbox. Both lists come from one native factory, so the hardware is probed once. Natively, the composite hardware factories implement WebRTC's QueryCodecSupport and report a format as power efficient when a hardware sub-factory supports it. CodecListExample marks each video codec as hardware or software with it. --- docs/guide/advanced/video-codecs.md | 18 +++++- docs/guide/examples.md | 2 +- .../webrtc/examples/CodecListExample.java | 59 ++++++------------- .../include/JNI_HardwareVideoDecoderFactory.h | 6 +- .../include/JNI_HardwareVideoEncoderFactory.h | 6 +- .../video/codec/DefaultVideoCodecFactories.h | 12 ++++ .../video/codec/HardwareVideoDecoderFactory.h | 9 +++ .../video/codec/HardwareVideoEncoderFactory.h | 11 ++++ .../src/JNI_HardwareVideoDecoderFactory.cpp | 16 ++++- .../src/JNI_HardwareVideoEncoderFactory.cpp | 16 ++++- .../codec/DefaultVideoCodecFactories.cpp | 47 +++++++++++++++ .../codec/HardwareVideoDecoderFactory.cpp | 17 ++++++ .../codec/HardwareVideoEncoderFactory.cpp | 17 ++++++ .../codec/HardwareVideoDecoderFactory.java | 35 ++++++++++- .../codec/HardwareVideoEncoderFactory.java | 35 ++++++++++- .../HardwareVideoEncoderIntegrationTest.java | 19 +++++- .../onvoid/webrtc/VideoCodecFactoryTests.java | 41 +++++++++++++ 17 files changed, 305 insertions(+), 61 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 455527303..6b44fcc4e 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -81,6 +81,22 @@ AV1 is decoded by VideoToolbox on Macs with an AV1 decoder, which are those with Which decoder a stream uses shows in the `decoderImplementation` statistic of its `inbound-rtp` stats, e.g. `MediaFoundation (Microsoft H264 Video Decoder MFT)`, `MediaFoundation (AV1VideoExtension)`, `VideoToolbox (VP9)`, `VideoToolbox (H.265)`, `VideoToolbox (AV1)` or `NVDEC (NVIDIA GeForce RTX 4070)`. +### Listing Hardware Codecs + +Both hardware factories tell which of their codecs the GPU of the machine handles. `getHardwareCodecs()` returns those of `getSupportedCodecs()`, in the same order, that a hardware encoder or decoder takes, as found when the factory was created; on macOS that includes H.264 through VideoToolbox. The other supported codecs are encoded or decoded in software: + +```java +HardwareVideoEncoderFactory encoders = new HardwareVideoEncoderFactory(); +List hardware = encoders.getHardwareCodecs(); + +for (VideoCodecInfo codec : encoders.getSupportedCodecs()) { + String where = hardware.contains(codec) ? "hardware" : "software"; + System.out.println(codec.getName() + " " + codec.getParameters() + ": " + where); +} +``` + +The list is per format: on Windows, for example, Media Foundation encoders take H.264 Constrained Baseline and Baseline with packetization mode 1, so the Main profile and packetization mode 0 are listed as software. A hardware codec is the one tried first, not a guarantee: a stream that falls back to software, as described above, or asks for what the hardware does not do, such as SVC, is encoded or decoded in software. The `powerEfficientEncoder` and `powerEfficientDecoder` statistics of the `outbound-rtp` and `inbound-rtp` stats tell what a running stream uses. + ### Native Codecs The encoders and decoders these factories create are `NativeVideoEncoder`s and `NativeVideoDecoder`s. They are placeholders that make WebRTC create the built-in codec, which then runs entirely inside WebRTC, so their methods are not to be called from Java. @@ -248,7 +264,7 @@ Whatever an encoder, decoder or factory method throws is logged and treated as ` - `VideoEncoderFactory`, `VideoDecoderFactory` — create the codecs of a factory. - `VideoEncoder`, `VideoDecoder`, `EncodedImage` — codecs implemented in Java. - `DefaultVideoEncoderFactory`, `DefaultVideoDecoderFactory` — the built-in codecs. -- `HardwareVideoEncoderFactory`, `HardwareVideoDecoderFactory` — the built-in codecs, on the GPU where possible. +- `HardwareVideoEncoderFactory`, `HardwareVideoDecoderFactory` — the built-in codecs, on the GPU where possible; `getHardwareCodecs()` lists those the GPU handles. - `RTCRtpTransceiver.setCodecPreferences()` — chooses among the negotiated codecs. For the full API, see the JavaDoc of the `dev.onvoid.webrtc.media.video.codec` package. diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 31fa27d2f..9d4ebc380 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -35,7 +35,7 @@ The [`CodecListExample`](https://github.com/devopvoid/webrtc-java/blob/master/we - Creating a `PeerConnectionFactory` with the `HardwareVideoEncoderFactory` and `HardwareVideoDecoderFactory` - Getting the supported codecs for both sending and receiving audio and video - Displaying detailed information about each codec -- Comparing the codecs of the default and the hardware video codec factories, and marking those only available in hardware (H.265) +- Listing the video codecs of the hardware factories and marking those the GPU encodes or decodes, with `getHardwareCodecs()` This example is useful for understanding what codecs are available on the current system, which can help with debugging compatibility issues or optimizing media quality. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java index 55d3665ff..40b586c19 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java @@ -23,8 +23,6 @@ import dev.onvoid.webrtc.RTCRtpCapabilities; import dev.onvoid.webrtc.RTCRtpCodecCapability; import dev.onvoid.webrtc.media.MediaType; -import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; -import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.VideoCodecInfo; @@ -35,15 +33,14 @@ * This example shows how to: *

*

- * The hardware factories support the codecs of the default factories, in the same order, and add - * those the GPU handles that have no software implementation (H.265). Whether a codec that both - * support is encoded or decoded on the GPU is decided per stream, and shows in the stats of a - * running connection ({@code powerEfficientEncoder} and {@code powerEfficientDecoder}). + * A hardware codec is tried first, not guaranteed: a stream falls back to software when the + * hardware fails or cannot do what the stream asks for. The stats of a running connection tell + * which one a stream uses ({@code powerEfficientEncoder} and {@code powerEfficientDecoder}). * * @author Alex Andres */ @@ -59,17 +56,17 @@ public static void main(String[] args) { .build(); try { - // Compare the video codecs of the default and the hardware factories. + // List the video codecs of the hardware factories and which of them the GPU handles. System.out.println("\nVIDEO CODEC FACTORIES:"); System.out.println("---------------------"); System.out.println("\nVideo Encoders:"); - printFactoryCodecs(new DefaultVideoEncoderFactory().getSupportedCodecs(), - hardwareEncoderFactory.getSupportedCodecs()); + printFactoryCodecs(hardwareEncoderFactory.getSupportedCodecs(), + hardwareEncoderFactory.getHardwareCodecs()); System.out.println("\nVideo Decoders:"); - printFactoryCodecs(new DefaultVideoDecoderFactory().getSupportedCodecs(), - hardwareDecoderFactory.getSupportedCodecs()); + printFactoryCodecs(hardwareDecoderFactory.getSupportedCodecs(), + hardwareDecoderFactory.getHardwareCodecs()); // Get receiver capabilities for audio and video. System.out.println("\nRECEIVER CAPABILITIES:"); @@ -102,21 +99,21 @@ public static void main(String[] args) { } /** - * Prints the codecs of a hardware video codec factory, marking those the default factory does - * not have, which are only available in hardware. + * Prints the codecs of a hardware video codec factory, marking each as encoded or decoded in + * hardware or in software. * - * @param defaultCodecs The codecs of the default factory. - * @param hardwareCodecs The codecs of the hardware factory. + * @param supportedCodecs The codecs the factory supports. + * @param hardwareCodecs The supported codecs the GPU handles. */ - private static void printFactoryCodecs(List defaultCodecs, + private static void printFactoryCodecs(List supportedCodecs, List hardwareCodecs) { - if (hardwareCodecs.isEmpty()) { + if (supportedCodecs.isEmpty()) { System.out.println(" No codecs found."); return; } - for (int i = 0; i < hardwareCodecs.size(); i++) { - VideoCodecInfo codec = hardwareCodecs.get(i); + for (int i = 0; i < supportedCodecs.size(); i++) { + VideoCodecInfo codec = supportedCodecs.get(i); StringBuilder sb = new StringBuilder(); sb.append(" Codec #").append(i + 1).append(": "); @@ -132,32 +129,12 @@ private static void printFactoryCodecs(List defaultCodecs, sb.append(" | Scalability Modes: ").append(scalabilityModes.size()); } - if (!containsCodec(defaultCodecs, codec)) { - sb.append(" | HARDWARE ONLY"); - } + sb.append(" | ").append(hardwareCodecs.contains(codec) ? "HARDWARE" : "SOFTWARE"); System.out.println(sb); } } - /** - * Checks whether the list holds a codec with the same name and format parameters. - * - * @param codecs The codecs to search. - * @param codec The codec to look for. - * - * @return true if the list holds the codec. - */ - private static boolean containsCodec(List codecs, VideoCodecInfo codec) { - for (VideoCodecInfo other : codecs) { - if (other.getName().equalsIgnoreCase(codec.getName()) - && other.getParameters().equals(codec.getParameters())) { - return true; - } - } - return false; - } - /** * Prints detailed information about each codec in the provided list. * All information for a codec is printed in a single line for conciseness. diff --git a/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h index 00a27e44d..c0c31d4b0 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoDecoderFactory.h @@ -24,10 +24,10 @@ extern "C" { #endif /* * Class: dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory - * Method: getSupportedCodecsInternal - * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + * Method: getCodecsInternal + * Signature: ()[[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; */ - JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory_getSupportedCodecsInternal + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory_getCodecsInternal (JNIEnv *, jclass); #ifdef __cplusplus diff --git a/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h index dba3732ed..34555ebe8 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_HardwareVideoEncoderFactory.h @@ -24,10 +24,10 @@ extern "C" { #endif /* * Class: dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory - * Method: getSupportedCodecsInternal - * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + * Method: getCodecsInternal + * Signature: ()[[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; */ - JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory_getSupportedCodecsInternal + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory_getCodecsInternal (JNIEnv *, jclass); #ifdef __cplusplus diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h index 67a7cacfa..b081c9df8 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h @@ -17,10 +17,12 @@ #ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_DEFAULT_VIDEO_CODEC_FACTORIES_H_ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_DEFAULT_VIDEO_CODEC_FACTORIES_H_ +#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include +#include namespace jni { @@ -39,6 +41,16 @@ namespace jni // where the platform has them, falling back to the software ones. On // macOS the default decoders already use VideoToolbox. std::unique_ptr CreateHardwareVideoDecoderFactory(); + + // The supported formats of the factory, in their order, that it encodes + // on the GPU first: those it reports as power efficient, and on macOS + // H.264, which the default encoders encode with VideoToolbox. + std::vector GetHardwareVideoEncoderFormats(const webrtc::VideoEncoderFactory & factory); + + // The supported formats of the factory, in their order, that it decodes + // on the GPU first, picked the way GetHardwareVideoEncoderFormats picks + // them. + std::vector GetHardwareVideoDecoderFormats(const webrtc::VideoDecoderFactory & factory); } #endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h index cb650b30f..ecfa1acc1 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h @@ -18,11 +18,13 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_HARDWARE_VIDEO_DECODER_FACTORY_H_ #include "api/environment/environment.h" +#include "api/video/resolution.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder_factory.h" #include +#include #include namespace jni @@ -43,6 +45,13 @@ namespace jni std::unique_ptr Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) override; + using webrtc::VideoDecoderFactory::QueryCodecSupport; + + // A supported format is power efficient when a hardware factory + // supports it, since that factory's decoder is the one tried first. + CodecSupport QueryCodecSupport(const webrtc::SdpVideoFormat & format, bool reference_scaling, + std::optional resolution) const override; + private: const std::vector> hardware; const std::unique_ptr software; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h index 5bd30e777..e1b4a2c39 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h @@ -18,11 +18,14 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_HARDWARE_VIDEO_ENCODER_FACTORY_H_ #include "api/environment/environment.h" +#include "api/video/resolution.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" #include +#include +#include #include namespace jni @@ -45,6 +48,14 @@ namespace jni std::unique_ptr Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) override; + using webrtc::VideoEncoderFactory::QueryCodecSupport; + + // A supported format is power efficient when a hardware factory + // supports it, since that factory's encoder is the one tried first. + CodecSupport QueryCodecSupport(const webrtc::SdpVideoFormat & format, + std::optional scalability_mode, + std::optional resolution) const override; + private: const std::vector> hardware; const std::unique_ptr software; diff --git a/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp index 251131e17..7342219ab 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoDecoderFactory.cpp @@ -19,13 +19,25 @@ #include "media/video/codec/VideoCodecInfo.h" #include "JavaUtils.h" -JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory_getSupportedCodecsInternal +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoDecoderFactory_getCodecsInternal (JNIEnv * env, jclass caller) { try { + // One factory for both lists, as making one probes the hardware. auto factory = jni::CreateHardwareVideoDecoderFactory(); - return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + auto supported = jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()); + auto hardware = jni::VideoCodecInfo::toJavaArray(env, jni::GetHardwareVideoDecoderFormats(*factory)); + auto arrayClass = jni::JavaLocalRef(env, env->GetObjectClass(supported.get())); + + jobjectArray codecs = env->NewObjectArray(2, arrayClass.get(), nullptr); + + ExceptionCheck(env); + + env->SetObjectArrayElement(codecs, 0, supported.get()); + env->SetObjectArrayElement(codecs, 1, hardware.get()); + + return codecs; } catch (...) { ThrowCxxJavaException(env); diff --git a/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp index f7cc8a122..93ff50cc6 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_HardwareVideoEncoderFactory.cpp @@ -19,13 +19,25 @@ #include "media/video/codec/VideoCodecInfo.h" #include "JavaUtils.h" -JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory_getSupportedCodecsInternal +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_HardwareVideoEncoderFactory_getCodecsInternal (JNIEnv * env, jclass caller) { try { + // One factory for both lists, as making one probes the hardware. auto factory = jni::CreateHardwareVideoEncoderFactory(); - return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + auto supported = jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()); + auto hardware = jni::VideoCodecInfo::toJavaArray(env, jni::GetHardwareVideoEncoderFormats(*factory)); + auto arrayClass = jni::JavaLocalRef(env, env->GetObjectClass(supported.get())); + + jobjectArray codecs = env->NewObjectArray(2, arrayClass.get(), nullptr); + + ExceptionCheck(env); + + env->SetObjectArrayElement(codecs, 0, supported.get()); + env->SetObjectArrayElement(codecs, 1, hardware.get()); + + return codecs; } catch (...) { ThrowCxxJavaException(env); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp index b0b6bce11..88e697ac5 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -19,6 +19,11 @@ #include "media/video/codec/HardwareVideoDecoderFactory.h" #include "media/video/codec/HardwareVideoEncoderFactory.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_codec.h" + +#include + #ifdef __APPLE__ #include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" #include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" @@ -39,6 +44,20 @@ namespace jni { + namespace + { + // WebRTC's Objective-C factories encode and decode H.264 with + // VideoToolbox, but do not report it as power efficient. + bool IsDefaultHardwareFormat(const webrtc::SdpVideoFormat & format) + { +#ifdef __APPLE__ + return webrtc::PayloadStringToCodecType(format.name) == webrtc::kVideoCodecH264; +#else + return false; +#endif + } + } + std::unique_ptr CreateDefaultVideoEncoderFactory() { #ifdef __APPLE__ @@ -86,4 +105,32 @@ namespace jni return std::make_unique(std::move(hardware), CreateDefaultVideoDecoderFactory()); } + + std::vector GetHardwareVideoEncoderFormats(const webrtc::VideoEncoderFactory & factory) + { + std::vector formats; + + for (const webrtc::SdpVideoFormat & format : factory.GetSupportedFormats()) { + if (IsDefaultHardwareFormat(format) + || factory.QueryCodecSupport(format, std::nullopt).is_power_efficient) { + formats.push_back(format); + } + } + + return formats; + } + + std::vector GetHardwareVideoDecoderFormats(const webrtc::VideoDecoderFactory & factory) + { + std::vector formats; + + for (const webrtc::SdpVideoFormat & format : factory.GetSupportedFormats()) { + if (IsDefaultHardwareFormat(format) + || factory.QueryCodecSupport(format, false).is_power_efficient) { + formats.push_back(format); + } + } + + return formats; + } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp index 64f2f0ea5..2c4ab18cf 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp @@ -62,6 +62,23 @@ namespace jni return formats; } + webrtc::VideoDecoderFactory::CodecSupport HardwareVideoDecoderFactory::QueryCodecSupport( + const webrtc::SdpVideoFormat & format, bool reference_scaling, + std::optional resolution) const + { + CodecSupport support = webrtc::VideoDecoderFactory::QueryCodecSupport(format, reference_scaling, resolution); + + if (!support.is_supported) { + return support; + } + + support.is_power_efficient = std::any_of(hardware.begin(), hardware.end(), [&](const auto & factory) { + return factory->QueryCodecSupport(format, reference_scaling, resolution).is_supported; + }) || software->QueryCodecSupport(format, reference_scaling, resolution).is_power_efficient; + + return support; + } + std::unique_ptr HardwareVideoDecoderFactory::Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) { diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp index 799114fe0..5af5985b7 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp @@ -62,6 +62,23 @@ namespace jni return formats; } + webrtc::VideoEncoderFactory::CodecSupport HardwareVideoEncoderFactory::QueryCodecSupport( + const webrtc::SdpVideoFormat & format, std::optional scalability_mode, + std::optional resolution) const + { + CodecSupport support = webrtc::VideoEncoderFactory::QueryCodecSupport(format, scalability_mode, resolution); + + if (!support.is_supported) { + return support; + } + + support.is_power_efficient = std::any_of(hardware.begin(), hardware.end(), [&](const auto & factory) { + return factory->QueryCodecSupport(format, scalability_mode, resolution).is_supported; + }) || software->QueryCodecSupport(format, scalability_mode, resolution).is_power_efficient; + + return support; + } + std::unique_ptr HardwareVideoEncoderFactory::Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) { diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 3321751db..1c28b939b 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -82,13 +82,17 @@ public final class HardwareVideoDecoderFactory implements VideoDecoderFactory { private final List supportedCodecs; + private final List hardwareCodecs; + /** * Creates a factory for the hardware video decoders. */ public HardwareVideoDecoderFactory() { - supportedCodecs = Collections.unmodifiableList( - Arrays.asList(getSupportedCodecsInternal())); + VideoCodecInfo[][] codecs = getCodecsInternal(); + + supportedCodecs = Collections.unmodifiableList(Arrays.asList(codecs[0])); + hardwareCodecs = Collections.unmodifiableList(Arrays.asList(codecs[1])); } @Override @@ -96,6 +100,27 @@ public List getSupportedCodecs() { return supportedCodecs; } + /** + * Returns the supported codecs that are decoded on the GPU, in the order + * of {@link #getSupportedCodecs()}. A codec counts when a hardware decoder + * of the machine takes it, as found when the factory was created; on + * macOS that includes H.264, which VideoToolbox decodes. The other + * supported codecs are decoded in software. + *

+ * A hardware decoder is tried first, not guaranteed: one that fails to + * start, or fails while decoding, is replaced by the software decoder of + * the same codec, and streams the hardware cannot decode, such as VP9 and AV1 with + * spatial layers, are decoded in software. The stats of a running + * connection tell which decoder a stream uses, in the + * {@code powerEfficientDecoder} attribute of its inbound RTP stats. + * + * @return The hardware codecs, unmodifiable; empty if the machine has no + * hardware decoder this library supports. + */ + public List getHardwareCodecs() { + return hardwareCodecs; + } + /** * Returns a built-in decoder for the given codec, decoding on the GPU if * it can. @@ -118,6 +143,10 @@ public NativeVideoDecoder createDecoder(VideoCodecInfo info) { return null; } - private static native VideoCodecInfo[] getSupportedCodecsInternal(); + /** + * Returns the supported codecs and the hardware codecs, from one native + * factory, as creating one probes the hardware. + */ + private static native VideoCodecInfo[][] getCodecsInternal(); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index d9990ebd7..5287c8b05 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -79,13 +79,17 @@ public final class HardwareVideoEncoderFactory implements VideoEncoderFactory { private final List supportedCodecs; + private final List hardwareCodecs; + /** * Creates a factory for the hardware video encoders. */ public HardwareVideoEncoderFactory() { - supportedCodecs = Collections.unmodifiableList( - Arrays.asList(getSupportedCodecsInternal())); + VideoCodecInfo[][] codecs = getCodecsInternal(); + + supportedCodecs = Collections.unmodifiableList(Arrays.asList(codecs[0])); + hardwareCodecs = Collections.unmodifiableList(Arrays.asList(codecs[1])); } @Override @@ -93,6 +97,27 @@ public List getSupportedCodecs() { return supportedCodecs; } + /** + * Returns the supported codecs that are encoded on the GPU, in the order + * of {@link #getSupportedCodecs()}. A codec counts when a hardware encoder + * of the machine takes it, as found when the factory was created; on + * macOS that includes H.264, which VideoToolbox encodes. The other + * supported codecs are encoded in software. + *

+ * A hardware encoder is tried first, not guaranteed: one that fails to + * start or gives up while encoding is replaced by the software encoder of + * the same codec, and a stream that asks for what the hardware does not + * do, such as SVC, is encoded in software. The stats of a running + * connection tell which encoder a stream uses, in the + * {@code powerEfficientEncoder} attribute of its outbound RTP stats. + * + * @return The hardware codecs, unmodifiable; empty if the machine has no + * hardware encoder this library supports. + */ + public List getHardwareCodecs() { + return hardwareCodecs; + } + /** * Returns a built-in encoder for the given codec, encoding on the GPU if * it can. @@ -115,6 +140,10 @@ public NativeVideoEncoder createEncoder(VideoCodecInfo info) { return null; } - private static native VideoCodecInfo[] getSupportedCodecsInternal(); + /** + * Returns the supported codecs and the hardware codecs, from one native + * factory, as creating one probes the hardware. + */ + private static native VideoCodecInfo[][] getCodecsInternal(); } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 23ec7a3d7..47fc78ffd 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -382,13 +382,15 @@ void hardwareEncodesH264() throws Exception { assumeTrue(OS.contains("win") || OS.contains("linux"), "hardware encoders are implemented on Windows and Linux only"); + HardwareVideoEncoderFactory encoders = new HardwareVideoEncoderFactory(); PeerConnectionFactory hardware = PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule) - .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .setVideoEncoderFactory(encoders) .build(); try { assertHardware(encoderImplementation(hardware, H264), HARDWARE_REQUIRED); + assertListedAsHardware(encoders, "H264"); } finally { hardware.dispose(); @@ -400,15 +402,17 @@ void hardwareEncodesAv1() throws Exception { assumeTrue(OS.contains("win") || OS.contains("linux"), "hardware encoders are implemented on Windows and Linux only"); + HardwareVideoEncoderFactory encoders = new HardwareVideoEncoderFactory(); PeerConnectionFactory hardware = PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule) - .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .setVideoEncoderFactory(encoders) .build(); try { // Frames arrive either way: from the GPU, or from libaom where the // GPU has no AV1 encoder. assertHardware(encoderImplementation(hardware, AV1), HARDWARE_AV1_REQUIRED); + assertListedAsHardware(encoders, "AV1"); } finally { hardware.dispose(); @@ -609,6 +613,17 @@ private static void assertHardware(String implementation, boolean required) { } } + /** + * Checks that a codec the GPU was seen to encode is among the hardware + * codecs of the factory. + */ + private static void assertListedAsHardware(HardwareVideoEncoderFactory factory, String codec) { + List hardware = factory.getHardwareCodecs(); + + assertTrue(hardware.stream().anyMatch(info -> codec.equalsIgnoreCase(info.getName())), + codec + " is not listed as hardware: " + hardware); + } + /** * Sends video in the preferred codec through a call until frames arrive, * and returns what the sender reports its encoder to be. diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java index 43614f0c2..d73410e81 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -34,6 +34,8 @@ import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.EncodedImage; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.NativeVideoEncoder; import dev.onvoid.webrtc.media.video.codec.VideoCodecInfo; import dev.onvoid.webrtc.media.video.codec.VideoCodecStatus; @@ -95,6 +97,17 @@ void defaultFactoriesListBuiltInCodecs() { assertTrue(decoders.contains("VP9"), decoders.toString()); } + @Test + void hardwareFactoriesListHardwareCodecs() { + HardwareVideoEncoderFactory encoders = new HardwareVideoEncoderFactory(); + HardwareVideoDecoderFactory decoders = new HardwareVideoDecoderFactory(); + + assertHardwareCodecs(new DefaultVideoEncoderFactory().getSupportedCodecs(), + encoders.getSupportedCodecs(), encoders.getHardwareCodecs()); + assertHardwareCodecs(new DefaultVideoDecoderFactory().getSupportedCodecs(), + decoders.getSupportedCodecs(), decoders.getHardwareCodecs()); + } + @Test void defaultFactoryCreatesNativeCodecs() { DefaultVideoEncoderFactory factory = new DefaultVideoEncoderFactory(); @@ -402,6 +415,34 @@ private PeerConnectionFactory testCodecFactory(VideoEncoderFactory encoderFactor .build(); } + /** + * Checks the hardware codecs of a hardware factory against its supported + * codecs and those of the matching default factory. Which codecs the + * hardware takes depends on the machine; that the codecs without a + * software implementation are among them does not. + */ + private static void assertHardwareCodecs(List software, + List supported, List hardware) { + // Supported codecs, in the same order. + int next = 0; + + for (VideoCodecInfo codec : hardware) { + int index = supported.subList(next, supported.size()).indexOf(codec); + + assertTrue(index >= 0, codec + " is not supported or out of order: " + supported); + + next += index + 1; + } + + for (VideoCodecInfo codec : supported) { + if (!software.contains(codec)) { + assertTrue(hardware.contains(codec), codec + " has no software codec: " + hardware); + } + } + + assertThrows(UnsupportedOperationException.class, () -> hardware.add(new VideoCodecInfo(TEST_CODEC))); + } + private static List names(List codecs) { return codecs.stream() .map(codec -> codec instanceof VideoCodecInfo From ef01d316b4ec09eecbcac83e2daa66c4eb8a1825 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 23:17:22 +0200 Subject: [PATCH 3/4] fix: keep H.264 High 4:4:4 off the hardware decoders NVDEC and the Media Foundation decoders offered every H.264 format WebRTC decodes, High 4:4:4 Predictive among them, though GPU decoders take 4:2:0 only. A 4:4:4 stream started on the GPU, failed, and lost frames until the next key frame reached the software decoder, and getHardwareCodecs() listed the profile as hardware. Both now offer the 4:2:0 profiles only, so 4:4:4 streams go to the software decoder from the start. --- docs/guide/advanced/video-codecs.md | 2 ++ .../include/media/video/codec/H264Format.h | 33 +++++++++++++++++ .../cpp/src/media/video/codec/H264Format.cpp | 36 +++++++++++++++++++ .../codec/nvdec/NvdecVideoDecoderFactory.cpp | 3 +- .../codec/windows/MFVideoDecoderFactory.cpp | 3 +- .../codec/HardwareVideoDecoderFactory.java | 3 +- .../onvoid/webrtc/VideoCodecFactoryTests.java | 12 +++++++ 7 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/H264Format.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/H264Format.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 6b44fcc4e..4f9460ae8 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -73,6 +73,8 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. +On Windows and Linux, H.264 is decoded in hardware in its 4:2:0 profiles. High 4:4:4 Predictive, which GPU decoders do not take, is decoded in software. + AV1 on Windows needs the *AV1 Video Extension*, which Windows 11 includes, and VP9 the *VP9 Video Extensions*, a free component of the Microsoft Store that a system may not have; without it, VP9 is decoded in software. H.265 needs the *HEVC Video Extensions* of the Microsoft Store, or an HEVC decoder of the GPU vendor; without one, H.265 is not offered, since it has no software decoder. Streams with spatial layers (SVC) are decoded in software too, since their layers reach a decoder without the index that says where each ends. Decoded frames are copied from GPU memory back to system memory, where WebRTC's frames are, so hardware decoding pays off mostly at high resolutions and with many streams; at low resolutions WebRTC's software decoders are about as cheap. A hardware decoder that fails, or turns out to decode in software, is replaced by the software decoder, which starts with the next key frame. On macOS, VP9 decoding in hardware saves CPU: in a measurement on an Apple M2 it took a fraction of the processor time libvpx needs, but multi-threaded libvpx was about as fast on the clock. VideoToolbox decodes VP9 in hardware only where the Mac has a decoder for it, which Apple silicon does; a Mac without one decodes VP9 with libvpx as before. Profile 2 (10 bit), sizes below 64x64 or above 4096x4096, and streams with spatial layers (SVC) are decoded with libvpx too. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/H264Format.h b/webrtc-jni/src/main/cpp/include/media/video/codec/H264Format.h new file mode 100644 index 000000000..97c6b6fd7 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/H264Format.h @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_H264_FORMAT_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_H264_FORMAT_H_ + +#include "api/video_codecs/sdp_video_format.h" + +#include + +namespace jni +{ + // The H.264 formats the hardware decoders offer: those WebRTC decodes, + // without High 4:4:4 Predictive. GPU decoders take 4:2:0 only, so a 4:4:4 + // stream goes to the software decoder from its start, instead of failing + // on the GPU and waiting for a key frame. + std::vector CreateH264HardwareDecoderFormats(); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/H264Format.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/H264Format.cpp new file mode 100644 index 000000000..d0f9b22c2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/H264Format.cpp @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/H264Format.h" + +#include "api/video_codecs/h264_profile_level_id.h" +#include "modules/video_coding/codecs/h264/include/h264.h" + +namespace jni +{ + std::vector CreateH264HardwareDecoderFormats() + { + std::vector formats = webrtc::SupportedH264DecoderCodecs(); + + std::erase_if(formats, [](const webrtc::SdpVideoFormat & format) { + const auto profileLevelId = webrtc::ParseSdpForH264ProfileLevelId(format.parameters); + + return !profileLevelId || profileLevelId->profile == webrtc::H264Profile::kProfilePredictiveHigh444; + }); + + return formats; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp index 3db8133e3..e87193469 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/nvdec/NvdecVideoDecoderFactory.cpp @@ -15,6 +15,7 @@ */ #include "media/video/codec/nvdec/NvdecVideoDecoderFactory.h" +#include "media/video/codec/H264Format.h" #include "media/video/codec/H265Format.h" #include "media/video/codec/nvdec/NvdecVideoDecoder.h" @@ -44,7 +45,7 @@ namespace jni std::vector formats; if (library.SupportsH264()) { - formats = webrtc::SupportedH264DecoderCodecs(); + formats = CreateH264HardwareDecoderFormats(); } if (library.SupportsH265()) { formats.push_back(CreateH265MainFormat()); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp index 36aa45182..76eaa8041 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/windows/MFVideoDecoderFactory.cpp @@ -15,6 +15,7 @@ */ #include "media/video/codec/windows/MFVideoDecoderFactory.h" +#include "media/video/codec/H264Format.h" #include "media/video/codec/H265Format.h" #include "media/video/codec/windows/MFDecoderUtils.h" #include "media/video/codec/windows/MFVideoDecoder.h" @@ -123,7 +124,7 @@ namespace jni std::vector formats; if (h264) { - formats = webrtc::SupportedH264DecoderCodecs(); + formats = CreateH264HardwareDecoderFormats(); } if (h265) { formats.push_back(CreateH265MainFormat()); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 1c28b939b..b19b291cc 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -61,7 +61,8 @@ * has not been tested on such a GPU yet. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in - * the same order. H.265 comes after them, only where the hardware decodes it: + * the same order. On Windows and Linux, H.264 High 4:4:4 Predictive, + * which GPU decoders do not take, is decoded in software. H.265 comes after them, only where the hardware decodes it: * there is no software decoder for it to fall back to, and it is never * decoded in software, so an H.265 stream the hardware cannot decode is not * decoded at all. diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java index d73410e81..1ecf9d5a5 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -108,6 +108,18 @@ void hardwareFactoriesListHardwareCodecs() { decoders.getSupportedCodecs(), decoders.getHardwareCodecs()); } + @Test + void hardwareDecodesNoH264High444() { + // GPU decoders take 4:2:0 only, so H.264 High 4:4:4 Predictive (profile + // 0xf4) stays with the software decoder. + for (VideoCodecInfo codec : new HardwareVideoDecoderFactory().getHardwareCodecs()) { + String profileLevelId = codec.getParameters().getOrDefault("profile-level-id", ""); + + assertFalse("H264".equalsIgnoreCase(codec.getName()) && profileLevelId.startsWith("f4"), + codec.toString()); + } + } + @Test void defaultFactoryCreatesNativeCodecs() { DefaultVideoEncoderFactory factory = new DefaultVideoEncoderFactory(); From 2c0d05926e95c3a8143af10bfe84a310b1c45a5b Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 23:23:01 +0200 Subject: [PATCH 4/4] perf: probe the video codec hardware once per process Every hardware codec factory probed the GPU anew, about 40 ms for the decoders on Windows, and a PeerConnectionFactory built with the hardware factories probed twice: once for their Java objects, and once more in the wrappers that took them through Java, where the two probes could also disagree. The platform's hardware factories, which hold nothing but what probing found, are now made once per process and setting of the hardware codec options, and shared. PeerConnectionFactory makes the hardware factories natively, as it does the default ones, without a detour through Java for every codec it creates. A Java factory of one's own makes the built-in factories only when it first hands out a native codec, so one that never does probes no hardware. --- docs/guide/advanced/video-codecs.md | 2 +- .../cpp/include/JNI_PeerConnectionFactory.h | 4 +- .../video/codec/DefaultVideoCodecFactories.h | 6 ++- .../video/codec/HardwareVideoDecoderFactory.h | 8 +++- .../video/codec/HardwareVideoEncoderFactory.h | 8 +++- .../video/codec/VideoDecoderFactoryWrapper.h | 12 +++++- .../video/codec/VideoEncoderFactoryWrapper.h | 12 +++++- .../cpp/src/JNI_PeerConnectionFactory.cpp | 13 ++++-- .../codec/DefaultVideoCodecFactories.cpp | 40 +++++++++++++++++-- .../codec/HardwareVideoDecoderFactory.cpp | 8 ++-- .../codec/HardwareVideoEncoderFactory.cpp | 8 ++-- .../codec/VideoDecoderFactoryWrapper.cpp | 20 +++++++--- .../codec/VideoEncoderFactoryWrapper.cpp | 20 +++++++--- .../onvoid/webrtc/PeerConnectionFactory.java | 40 +++++++++++++------ .../codec/HardwareVideoDecoderFactory.java | 8 ++-- .../codec/HardwareVideoEncoderFactory.java | 8 ++-- .../HardwareVideoEncoderIntegrationTest.java | 35 ++++++++++++++++ .../onvoid/webrtc/VideoCodecFactoryTests.java | 38 ++++++++++++++++++ 18 files changed, 232 insertions(+), 58 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 4f9460ae8..962a9de62 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -85,7 +85,7 @@ Which decoder a stream uses shows in the `decoderImplementation` statistic of it ### Listing Hardware Codecs -Both hardware factories tell which of their codecs the GPU of the machine handles. `getHardwareCodecs()` returns those of `getSupportedCodecs()`, in the same order, that a hardware encoder or decoder takes, as found when the factory was created; on macOS that includes H.264 through VideoToolbox. The other supported codecs are encoded or decoded in software: +Both hardware factories tell which of their codecs the GPU of the machine handles. `getHardwareCodecs()` returns those of `getSupportedCodecs()`, in the same order, that a hardware encoder or decoder takes; on macOS that includes H.264 through VideoToolbox. The other supported codecs are encoded or decoded in software. The hardware is probed once per process, which every factory then shares, and again only when one of the system properties above is set differently; a GPU, driver or codec extension installed while the process runs is seen by the next one: ```java HardwareVideoEncoderFactory encoders = new HardwareVideoEncoderFactory(); diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index c5b40c021..5881dccd0 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -82,10 +82,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory * Method: initialize - * Signature: (Ljava/util/Map;Ldev/onvoid/webrtc/media/audio/AudioDeviceModuleBase;Ldev/onvoid/webrtc/media/audio/AudioProcessing;Ldev/onvoid/webrtc/media/video/codec/VideoEncoderFactory;Ldev/onvoid/webrtc/media/video/codec/VideoDecoderFactory;)V + * Signature: (Ljava/util/Map;Ldev/onvoid/webrtc/media/audio/AudioDeviceModuleBase;Ldev/onvoid/webrtc/media/audio/AudioProcessing;Ldev/onvoid/webrtc/media/video/codec/VideoEncoderFactory;Ldev/onvoid/webrtc/media/video/codec/VideoDecoderFactory;ZZ)V */ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize - (JNIEnv *, jobject, jobject, jobject, jobject, jobject, jobject); + (JNIEnv *, jobject, jobject, jobject, jobject, jobject, jobject, jboolean, jboolean); #ifdef __cplusplus } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h index b081c9df8..97e90e7d5 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h @@ -33,13 +33,15 @@ namespace jni // The default encoders, with the encoders of the GPU in front of them // where the platform has them, falling back to the software ones. On - // macOS the default encoders already use VideoToolbox. + // macOS the default encoders already use VideoToolbox. The hardware is + // probed once per process and setting of the hardware codec options. std::unique_ptr CreateHardwareVideoEncoderFactory(); std::unique_ptr CreateDefaultVideoDecoderFactory(); // The default decoders, with the decoders of the GPU in front of them // where the platform has them, falling back to the software ones. On - // macOS the default decoders already use VideoToolbox. + // macOS the default decoders already use VideoToolbox. The hardware is + // probed once per process and setting of the hardware codec options. std::unique_ptr CreateHardwareVideoDecoderFactory(); // The supported formats of the factory, in their order, that it encodes diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h index ecfa1acc1..02ca2e9ff 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoDecoderFactory.h @@ -36,8 +36,12 @@ namespace jni class HardwareVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: + // The hardware factories. They hold nothing but what probing for + // them found, so several factories, on any thread, can share them. + using HardwareFactories = std::vector>; + // The hardware factories are in order of preference. - HardwareVideoDecoderFactory(std::vector> hardware, + HardwareVideoDecoderFactory(std::shared_ptr hardware, std::unique_ptr software); ~HardwareVideoDecoderFactory() override = default; @@ -53,7 +57,7 @@ namespace jni std::optional resolution) const override; private: - const std::vector> hardware; + const std::shared_ptr hardware; const std::unique_ptr software; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h index e1b4a2c39..825cc5d57 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareVideoEncoderFactory.h @@ -39,8 +39,12 @@ namespace jni class HardwareVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: + // The hardware factories. They hold nothing but what probing for + // them found, so several factories, on any thread, can share them. + using HardwareFactories = std::vector>; + // The hardware factories are in order of preference. - HardwareVideoEncoderFactory(std::vector> hardware, + HardwareVideoEncoderFactory(std::shared_ptr hardware, std::unique_ptr software); ~HardwareVideoEncoderFactory() override = default; @@ -57,7 +61,7 @@ namespace jni std::optional resolution) const override; private: - const std::vector> hardware; + const std::shared_ptr hardware; const std::unique_ptr software; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h index 3d507d857..c16738016 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h @@ -28,6 +28,7 @@ #include #include +#include #include namespace jni @@ -48,6 +49,12 @@ namespace jni std::unique_ptr Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) override; + private: + // The built-in decoders, with the GPU's in front if hardware is + // true, made the first time a NativeVideoDecoder asks for them: + // a factory that never hands one out probes no hardware. + webrtc::VideoDecoderFactory & BuiltInFactory(bool hardware); + private: class JavaVideoDecoderFactoryClass : public JavaClass { @@ -66,8 +73,9 @@ namespace jni const JavaGlobalRef factory; const std::shared_ptr javaClass; // The built-in decoders, and those with the GPU's in front. - const std::unique_ptr defaultFactory; - const std::unique_ptr hardwareFactory; + std::mutex builtInMutex; + std::unique_ptr defaultFactory; + std::unique_ptr hardwareFactory; std::vector supportedFormats; }; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h index 663d2f61f..aa517c4a2 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h @@ -28,6 +28,7 @@ #include #include +#include #include namespace jni @@ -48,6 +49,12 @@ namespace jni std::unique_ptr Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) override; + private: + // The built-in encoders, with the GPU's in front if hardware is + // true, made the first time a NativeVideoEncoder asks for them: + // a factory that never hands one out probes no hardware. + webrtc::VideoEncoderFactory & BuiltInFactory(bool hardware); + private: class JavaVideoEncoderFactoryClass : public JavaClass { @@ -66,8 +73,9 @@ namespace jni const JavaGlobalRef factory; const std::shared_ptr javaClass; // The built-in encoders, and those with the GPU's in front. - const std::unique_ptr defaultFactory; - const std::unique_ptr hardwareFactory; + std::mutex builtInMutex; + std::unique_ptr defaultFactory; + std::unique_ptr hardwareFactory; std::vector supportedFormats; }; diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 2ed4de40d..0c4a8b5ff 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -55,7 +55,7 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize (JNIEnv * env, jobject caller, jobject jFieldTrials, jobject audioModule, jobject audioProcessing, - jobject jVideoEncoderFactory, jobject jVideoDecoderFactory) + jobject jVideoEncoderFactory, jobject jVideoDecoderFactory, jboolean hardwareEncoders, jboolean hardwareDecoders) { webrtc::AudioDeviceModule * audioDevModule = (audioModule != nullptr) ? GetHandle(env, audioModule) @@ -82,13 +82,18 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize : std::make_unique(std::move(fieldTrialsMap)); // Asks the Java factories for their codecs, which is where a broken - // factory surfaces, before anything else is set up. + // factory surfaces, before anything else is set up. Without a Java + // factory, the built-in codecs are used directly. std::unique_ptr videoEncoderFactory = (jVideoEncoderFactory != nullptr) ? std::make_unique(env, jVideoEncoderFactory) - : jni::CreateDefaultVideoEncoderFactory(); + : hardwareEncoders + ? jni::CreateHardwareVideoEncoderFactory() + : jni::CreateDefaultVideoEncoderFactory(); std::unique_ptr videoDecoderFactory = (jVideoDecoderFactory != nullptr) ? std::make_unique(env, jVideoDecoderFactory) - : jni::CreateDefaultVideoDecoderFactory(); + : hardwareDecoders + ? jni::CreateHardwareVideoDecoderFactory() + : jni::CreateDefaultVideoDecoderFactory(); auto networkThread = webrtc::Thread::CreateWithSocketServer(); networkThread->SetName("webrtc_jni_network_thread", nullptr); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp index 88e697ac5..fee88d766 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -16,13 +16,17 @@ #include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/HardwareCodecOptions.h" #include "media/video/codec/HardwareVideoDecoderFactory.h" #include "media/video/codec/HardwareVideoEncoderFactory.h" #include "api/video/video_codec_type.h" #include "api/video_codecs/video_codec.h" +#include +#include #include +#include #ifdef __APPLE__ #include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" @@ -56,6 +60,34 @@ namespace jni return false; #endif } + + // Probes for the hardware factories of the platform once for each + // setting of the options that change what is found, and shares what + // it found: a probe takes tens of milliseconds, and so every factory + // works with the same codecs. A GPU, driver or codec installed or + // removed later is noticed only by a new process. + template + std::shared_ptr GetPlatformHardwareFactories(Factories (*probe)()) + { + using Options = std::pair; + + static std::mutex mutex; + // Never destroyed: the factories use libraries that may be gone + // by the time static objects are destroyed as the process exits. + static auto * cache = new std::map>(); + + const Options options(IsHardwareVp9EncoderEnabled(), IsExperimentalVaapiEnabled()); + + std::lock_guard lock(mutex); + + std::shared_ptr & factories = (*cache)[options]; + + if (!factories) { + factories = std::make_shared(probe()); + } + + return factories; + } } std::unique_ptr CreateDefaultVideoEncoderFactory() @@ -73,9 +105,9 @@ namespace jni std::unique_ptr CreateHardwareVideoEncoderFactory() { - std::vector> hardware = CreatePlatformHardwareVideoEncoderFactories(); + auto hardware = GetPlatformHardwareFactories(&CreatePlatformHardwareVideoEncoderFactories); - if (hardware.empty()) { + if (hardware->empty()) { return CreateDefaultVideoEncoderFactory(); } @@ -97,9 +129,9 @@ namespace jni std::unique_ptr CreateHardwareVideoDecoderFactory() { - std::vector> hardware = CreatePlatformHardwareVideoDecoderFactories(); + auto hardware = GetPlatformHardwareFactories(&CreatePlatformHardwareVideoDecoderFactories); - if (hardware.empty()) { + if (hardware->empty()) { return CreateDefaultVideoDecoderFactory(); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp index 2c4ab18cf..38dac83ab 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoDecoderFactory.cpp @@ -34,7 +34,7 @@ namespace jni } HardwareVideoDecoderFactory::HardwareVideoDecoderFactory( - std::vector> hardware, + std::shared_ptr hardware, std::unique_ptr software) : hardware(std::move(hardware)), software(std::move(software)) @@ -47,7 +47,7 @@ namespace jni // does not change what a peer connection negotiates. std::vector formats = software->GetSupportedFormats(); - for (const auto & factory : hardware) { + for (const auto & factory : *hardware) { for (const webrtc::SdpVideoFormat & format : factory->GetSupportedFormats()) { bool listed = std::any_of(formats.begin(), formats.end(), [&](const webrtc::SdpVideoFormat & other) { return other.IsSameCodec(format); @@ -72,7 +72,7 @@ namespace jni return support; } - support.is_power_efficient = std::any_of(hardware.begin(), hardware.end(), [&](const auto & factory) { + support.is_power_efficient = std::any_of(hardware->begin(), hardware->end(), [&](const auto & factory) { return factory->QueryCodecSupport(format, reference_scaling, resolution).is_supported; }) || software->QueryCodecSupport(format, reference_scaling, resolution).is_power_efficient; @@ -90,7 +90,7 @@ namespace jni // Built from the back, so that the most preferred decoder comes first // and each one falls back to the chain behind it. - for (auto factory = hardware.rbegin(); factory != hardware.rend(); ++factory) { + for (auto factory = hardware->rbegin(); factory != hardware->rend(); ++factory) { if (!Supports(**factory, format)) { continue; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp index 5af5985b7..ec159ddeb 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareVideoEncoderFactory.cpp @@ -34,7 +34,7 @@ namespace jni } HardwareVideoEncoderFactory::HardwareVideoEncoderFactory( - std::vector> hardware, + std::shared_ptr hardware, std::unique_ptr software) : hardware(std::move(hardware)), software(std::move(software)) @@ -47,7 +47,7 @@ namespace jni // does not change which codec a peer connection prefers. std::vector formats = software->GetSupportedFormats(); - for (const auto & factory : hardware) { + for (const auto & factory : *hardware) { for (const webrtc::SdpVideoFormat & format : factory->GetSupportedFormats()) { bool listed = std::any_of(formats.begin(), formats.end(), [&](const webrtc::SdpVideoFormat & other) { return other.IsSameCodec(format); @@ -72,7 +72,7 @@ namespace jni return support; } - support.is_power_efficient = std::any_of(hardware.begin(), hardware.end(), [&](const auto & factory) { + support.is_power_efficient = std::any_of(hardware->begin(), hardware->end(), [&](const auto & factory) { return factory->QueryCodecSupport(format, scalability_mode, resolution).is_supported; }) || software->QueryCodecSupport(format, scalability_mode, resolution).is_power_efficient; @@ -90,7 +90,7 @@ namespace jni // Built from the back, so that the most preferred encoder comes first // and each one falls back to the chain behind it. - for (auto factory = hardware.rbegin(); factory != hardware.rend(); ++factory) { + for (auto factory = hardware->rbegin(); factory != hardware->rend(); ++factory) { if (!Supports(**factory, format)) { continue; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp index cbb9211cb..ce5bdd671 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp @@ -30,9 +30,7 @@ namespace jni { VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper(JNIEnv * env, jobject factory) : factory(env, factory), - javaClass(JavaClasses::get(env)), - defaultFactory(CreateDefaultVideoDecoderFactory()), - hardwareFactory(CreateHardwareVideoDecoderFactory()) + javaClass(JavaClasses::get(env)) { JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); @@ -71,9 +69,8 @@ namespace jni JavaLocalRef nativeInfo(env, env->GetObjectField(decoder, javaClass->nativeDecoderCodecInfo)); const bool hardware = env->GetBooleanField(decoder, javaClass->nativeDecoderHardwareAcceleration) == JNI_TRUE; - webrtc::VideoDecoderFactory & builtIn = hardware ? *hardwareFactory : *defaultFactory; - return builtIn.Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + return BuiltInFactory(hardware).Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); } return std::make_unique(env, decoder); @@ -87,6 +84,19 @@ namespace jni return nullptr; } + webrtc::VideoDecoderFactory & VideoDecoderFactoryWrapper::BuiltInFactory(bool hardware) + { + std::lock_guard lock(builtInMutex); + + std::unique_ptr & factory = hardware ? hardwareFactory : defaultFactory; + + if (!factory) { + factory = hardware ? CreateHardwareVideoDecoderFactory() : CreateDefaultVideoDecoderFactory(); + } + + return *factory; + } + VideoDecoderFactoryWrapper::JavaVideoDecoderFactoryClass::JavaVideoDecoderFactoryClass(JNIEnv * env) { jclass cls = FindClass(env, PKG_CODEC"VideoDecoderFactory"); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp index 2e2a4b125..9bcdd31d3 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp @@ -30,9 +30,7 @@ namespace jni { VideoEncoderFactoryWrapper::VideoEncoderFactoryWrapper(JNIEnv * env, jobject factory) : factory(env, factory), - javaClass(JavaClasses::get(env)), - defaultFactory(CreateDefaultVideoEncoderFactory()), - hardwareFactory(CreateHardwareVideoEncoderFactory()) + javaClass(JavaClasses::get(env)) { JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); @@ -70,9 +68,8 @@ namespace jni if (env->IsInstanceOf(encoder, javaClass->nativeEncoderClass)) { JavaLocalRef nativeInfo(env, env->GetObjectField(encoder, javaClass->nativeEncoderCodecInfo)); const bool hardware = env->GetBooleanField(encoder, javaClass->nativeEncoderHardwareAcceleration) == JNI_TRUE; - webrtc::VideoEncoderFactory & builtIn = hardware ? *hardwareFactory : *defaultFactory; - return builtIn.Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + return BuiltInFactory(hardware).Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); } return std::make_unique(env, encoder, format); @@ -86,6 +83,19 @@ namespace jni return nullptr; } + webrtc::VideoEncoderFactory & VideoEncoderFactoryWrapper::BuiltInFactory(bool hardware) + { + std::lock_guard lock(builtInMutex); + + std::unique_ptr & factory = hardware ? hardwareFactory : defaultFactory; + + if (!factory) { + factory = hardware ? CreateHardwareVideoEncoderFactory() : CreateDefaultVideoEncoderFactory(); + } + + return *factory; + } + VideoEncoderFactoryWrapper::JavaVideoEncoderFactoryClass::JavaVideoEncoderFactoryClass(JNIEnv * env) { jclass cls = FindClass(env, PKG_CODEC"VideoEncoderFactory"); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index d95741e0b..bc2a9e391 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -30,6 +30,8 @@ import dev.onvoid.webrtc.media.video.VideoTrack; import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; @@ -107,7 +109,7 @@ private enum AudioInputMode { * Creates an instance of PeerConnectionFactory. */ public PeerConnectionFactory() { - initialize(null, null, null, null, null); + initialize(null, null, null, null, null, false, false); } /** @@ -117,7 +119,7 @@ public PeerConnectionFactory() { * @param audioProcessing The custom audio processing module. */ public PeerConnectionFactory(AudioProcessing audioProcessing) { - initialize(null, null, audioProcessing, null, null); + initialize(null, null, audioProcessing, null, null, false, false); } /** @@ -127,7 +129,7 @@ public PeerConnectionFactory(AudioProcessing audioProcessing) { * @param audioModule The custom audio device module. */ public PeerConnectionFactory(AudioDeviceModuleBase audioModule) { - initialize(null, audioModule, null, null, null); + initialize(null, audioModule, null, null, null, false, false); } /** @@ -139,7 +141,7 @@ public PeerConnectionFactory(AudioDeviceModuleBase audioModule) { */ public PeerConnectionFactory(AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing) { - initialize(null, audioModule, audioProcessing, null, null); + initialize(null, audioModule, audioProcessing, null, null, false, false); } /** @@ -151,7 +153,7 @@ public PeerConnectionFactory(AudioDeviceModuleBase audioModule, * non-null and non-empty. */ public PeerConnectionFactory(Map fieldTrials) { - initialize(fieldTrials, null, null, null, null); + initialize(fieldTrials, null, null, null, null, false, false); } /** @@ -165,7 +167,7 @@ public PeerConnectionFactory(Map fieldTrials) { */ public PeerConnectionFactory(Map fieldTrials, AudioProcessing audioProcessing) { - initialize(fieldTrials, null, audioProcessing, null, null); + initialize(fieldTrials, null, audioProcessing, null, null, false, false); } /** @@ -179,7 +181,7 @@ public PeerConnectionFactory(Map fieldTrials, */ public PeerConnectionFactory(Map fieldTrials, AudioDeviceModuleBase audioModule) { - initialize(fieldTrials, audioModule, null, null, null); + initialize(fieldTrials, audioModule, null, null, null, false, false); } /** @@ -194,7 +196,7 @@ public PeerConnectionFactory(Map fieldTrials, */ public PeerConnectionFactory(Map fieldTrials, AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing) { - initialize(fieldTrials, audioModule, audioProcessing, null, null); + initialize(fieldTrials, audioModule, audioProcessing, null, null, false, false); } /** @@ -204,16 +206,22 @@ public PeerConnectionFactory(Map fieldTrials, * @param builder The builder with the modules to use. */ private PeerConnectionFactory(Builder builder) { - // The built-in codecs need no detour through Java. - VideoEncoderFactory encoderFactory = builder.videoEncoderFactory instanceof DefaultVideoEncoderFactory + // The built-in codecs, with or without the GPU's, need no detour + // through Java; the native factories are made directly. + boolean hardwareEncoders = builder.videoEncoderFactory instanceof HardwareVideoEncoderFactory; + boolean hardwareDecoders = builder.videoDecoderFactory instanceof HardwareVideoDecoderFactory; + + VideoEncoderFactory encoderFactory = hardwareEncoders + || builder.videoEncoderFactory instanceof DefaultVideoEncoderFactory ? null : builder.videoEncoderFactory; - VideoDecoderFactory decoderFactory = builder.videoDecoderFactory instanceof DefaultVideoDecoderFactory + VideoDecoderFactory decoderFactory = hardwareDecoders + || builder.videoDecoderFactory instanceof DefaultVideoDecoderFactory ? null : builder.videoDecoderFactory; initialize(builder.fieldTrials, builder.audioModule, builder.audioProcessing, - encoderFactory, decoderFactory); + encoderFactory, decoderFactory, hardwareEncoders, hardwareDecoders); } /** @@ -397,10 +405,16 @@ public RTCPeerConnection createPeerConnection(RTCConfiguration config, @Override public native void dispose(); + /** + * Creates the native factory. A null codec factory stands for the + * built-in codecs: those with the GPU's in front when the matching + * {@code hardware} flag is set, the default ones otherwise. + */ private native void initialize(Map fieldTrials, AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing, VideoEncoderFactory videoEncoderFactory, - VideoDecoderFactory videoDecoderFactory); + VideoDecoderFactory videoDecoderFactory, + boolean hardwareEncoders, boolean hardwareDecoders); private native AudioTrackSource createAudioSourceInternal(AudioOptions options); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index b19b291cc..9023deaf9 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -104,9 +104,11 @@ public List getSupportedCodecs() { /** * Returns the supported codecs that are decoded on the GPU, in the order * of {@link #getSupportedCodecs()}. A codec counts when a hardware decoder - * of the machine takes it, as found when the factory was created; on - * macOS that includes H.264, which VideoToolbox decodes. The other - * supported codecs are decoded in software. + * of the machine takes it; on macOS that includes H.264, which + * VideoToolbox decodes. The other supported codecs are decoded in + * software. The hardware is probed once per process, and again only for + * a new setting of the system properties that turn hardware codecs on, so + * a GPU or driver that changes later is seen by a new process only. *

* A hardware decoder is tried first, not guaranteed: one that fails to * start, or fails while decoding, is replaced by the software decoder of diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 5287c8b05..77bb58655 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -100,9 +100,11 @@ public List getSupportedCodecs() { /** * Returns the supported codecs that are encoded on the GPU, in the order * of {@link #getSupportedCodecs()}. A codec counts when a hardware encoder - * of the machine takes it, as found when the factory was created; on - * macOS that includes H.264, which VideoToolbox encodes. The other - * supported codecs are encoded in software. + * of the machine takes it; on macOS that includes H.264, which + * VideoToolbox encodes. The other supported codecs are encoded in + * software. The hardware is probed once per process, and again only for + * a new setting of the system properties that turn hardware codecs on, so + * a GPU or driver that changes later is seen by a new process only. *

* A hardware encoder is tried first, not guaranteed: one that fails to * start or gives up while encoding is replaced by the software encoder of diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 47fc78ffd..dc447ffa4 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -36,6 +36,8 @@ import dev.onvoid.webrtc.media.video.codec.VideoCodecStatus; import dev.onvoid.webrtc.media.video.codec.VideoDecoder; import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoEncoder; +import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -397,6 +399,39 @@ void hardwareEncodesH264() throws Exception { } } + @Test + void delegatingFactoryEncodesInHardware() throws Exception { + // A factory of one's own that hands out the hardware encoders goes + // through Java, unlike the hardware factory itself, and gets the + // built-in hardware encoders made the first time one is asked for. + assumeTrue(OS.contains("win") || OS.contains("linux"), + "hardware encoders are implemented on Windows and Linux only"); + + HardwareVideoEncoderFactory builtIn = new HardwareVideoEncoderFactory(); + PeerConnectionFactory delegating = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new VideoEncoderFactory() { + + @Override + public List getSupportedCodecs() { + return builtIn.getSupportedCodecs(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return builtIn.createEncoder(info); + } + }) + .build(); + + try { + assertHardware(encoderImplementation(delegating, H264), HARDWARE_REQUIRED); + } + finally { + delegating.dispose(); + } + } + @Test void hardwareEncodesAv1() throws Exception { assumeTrue(OS.contains("win") || OS.contains("linux"), diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java index 1ecf9d5a5..e7bc1ac22 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -51,6 +51,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -85,6 +86,9 @@ class VideoCodecFactoryTests extends TestBase { /** What precedes the payload in the slice of an H.265 test access unit. */ private static final String H265_MARKER = "WRTC:"; + /** The RTP capabilities WebRTC adds that are no media codec. */ + private static final List RTP_ONLY_CODECS = Arrays.asList("rtx", "red", "ulpfec", "flexfec-03"); + @Test void defaultFactoriesListBuiltInCodecs() { @@ -108,6 +112,30 @@ void hardwareFactoriesListHardwareCodecs() { decoders.getSupportedCodecs(), decoders.getHardwareCodecs()); } + @Test + void builtFactoryUsesHardwareCodecs() { + // The hardware factories are made natively, without their Java + // objects; what is negotiated has to be what those list. + HardwareVideoEncoderFactory encoders = new HardwareVideoEncoderFactory(); + HardwareVideoDecoderFactory decoders = new HardwareVideoDecoderFactory(); + + PeerConnectionFactory built = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(encoders) + .setVideoDecoderFactory(decoders) + .build(); + + try { + assertEquals(names(encoders.getSupportedCodecs()), + mediaCodecNames(built.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs())); + assertEquals(names(decoders.getSupportedCodecs()), + mediaCodecNames(built.getRtpReceiverCapabilities(MediaType.VIDEO).getCodecs())); + } + finally { + built.dispose(); + } + } + @Test void hardwareDecodesNoH264High444() { // GPU decoders take 4:2:0 only, so H.264 High 4:4:4 Predictive (profile @@ -455,6 +483,16 @@ private static void assertHardwareCodecs(List software, assertThrows(UnsupportedOperationException.class, () -> hardware.add(new VideoCodecInfo(TEST_CODEC))); } + /** + * The names of the media codecs among RTP capabilities, without those + * WebRTC adds for retransmission, redundancy and error correction. + */ + private static List mediaCodecNames(List capabilities) { + return names(capabilities).stream() + .filter(name -> !RTP_ONLY_CODECS.contains(name.toLowerCase(Locale.ROOT))) + .collect(Collectors.toList()); + } + private static List names(List codecs) { return codecs.stream() .map(codec -> codec instanceof VideoCodecInfo