From fbdd24245d77be482fe4e88d39f6de864930a25d Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 4 Oct 2026 17:48:01 +0200 Subject: [PATCH 1/2] perf: reuse the buffers of hardware-decoded frames in the media player For every picture the hardware path dropped the system-memory frame the picture was read back into, and allocated a new I420 frame for the conversion. At 4K that is two fresh allocations of 12 MB per picture, and a fresh allocation costs more to fault in than the copy into it. The read-back frame now stays between pictures, with a new buffer only when the size or the pixel format changes (the transfer copies no more than the smaller of the two frames), and the I420 frames come from a buffer pool, so a picture lands in memory that is already mapped. The conversion of other formats in software uses the pool as well. CPU cycles on the decoding thread per picture, on an NVIDIA GeForce RTX 3080 through Direct3D 11: the read-back went from 11.0 to 5.3 million at 4K and from 3.3 to 1.8 million at 1080p, the conversion from 10.4 to 5.7 and from 2.2 to 1.3. --- .../src/main/cpp/include/media/VideoDecoder.h | 11 ++ .../src/main/cpp/src/media/VideoDecoder.cpp | 104 +++++++++++++++--- 2 files changed, 99 insertions(+), 16 deletions(-) diff --git a/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h index 2660affb4..5c1a22a20 100644 --- a/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h +++ b/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h @@ -120,6 +120,11 @@ namespace ffmpeg // reference the caller owns. int ConvertToI420(const AVFrame * source, AVFrame ** result); + // A new I420 frame of the given size, whose buffer comes from a + // pool, so that the picture lands in memory that is already + // mapped. The caller owns the frame. + int AllocateI420(int width, int height, AVFrame ** result); + // A picture decoded while the packets of a failed hardware decoder // were decoded again, which the caller has not asked for yet. struct Pending @@ -133,6 +138,12 @@ namespace ffmpeg SwsContext * sws_context_ = nullptr; AVFrame * decoded_ = nullptr; AVFrame * transferred_ = nullptr; + // The size of the decoder's surfaces that transferred_ was made for. + int transfer_width_ = 0; + int transfer_height_ = 0; + AVBufferPool * pool_ = nullptr; + int pool_width_ = 0; + int pool_height_ = 0; AVRational time_base_ = { 0, 1 }; AVPixelFormat hardware_format_ = AV_PIX_FMT_NONE; diff --git a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp index c1515a36f..142f236f2 100644 --- a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -232,6 +232,14 @@ namespace ffmpeg sws_context_ = nullptr; } + + // Frames still out with WebRTC keep their buffers, and the pool goes + // when the last of them comes back. + av_buffer_pool_uninit(&pool_); + + pool_width_ = 0; + pool_height_ = 0; + if (codec_context_ != nullptr) { avcodec_free_context(&codec_context_); } @@ -326,18 +334,35 @@ namespace ffmpeg // The picture is in the memory of the hardware decoder. Bring it // into system memory, as NV12, and convert it like any other // format that is not I420. - av_frame_unref(transferred_); + // + // The system-memory frame stays between pictures: a buffer of + // this size is a fresh allocation every time it is dropped, and + // the pages of a fresh allocation cost more to fault in than the + // copy into them. The buffer is made for the size of the decoder's + // surfaces, which is as much as a transfer may copy (Direct3D 12 + // copies all of it, padding included), so it is kept only for + // as long as those, the picture and the format stay as they are. + const AVHWFramesContext * frames = reinterpret_cast(decoded_->hw_frames_ctx->data); + + if (transferred_->buf[0] != nullptr && (transferred_->width != decoded_->width + || transferred_->height != decoded_->height || transferred_->format != frames->sw_format + || transfer_width_ != frames->width || transfer_height_ != frames->height)) { + av_frame_unref(transferred_); + } + + const bool allocated = transferred_->buf[0] == nullptr; result = av_hwframe_transfer_data(transferred_, decoded_, 0); if (result >= 0) { - transferred_->width = decoded_->width; - transferred_->height = decoded_->height; + if (allocated) { + transfer_width_ = frames->width; + transfer_height_ = frames->height; + } result = ConvertToI420(transferred_, frame); } - av_frame_unref(transferred_); av_frame_unref(decoded_); } else if (IsI420(decoded_->format)) { @@ -485,21 +510,11 @@ namespace ffmpeg return AVERROR(EINVAL); } - AVFrame * converted = av_frame_alloc(); + AVFrame * converted = nullptr; - if (converted == nullptr) { - return AVERROR(ENOMEM); - } - - converted->format = AV_PIX_FMT_YUV420P; - converted->width = source->width; - converted->height = source->height; - - int error = av_frame_get_buffer(converted, 0); + int error = AllocateI420(source->width, source->height, &converted); if (error < 0) { - av_frame_free(&converted); - return error; } @@ -517,6 +532,63 @@ namespace ffmpeg return 0; } + int VideoDecoder::AllocateI420(int width, int height, AVFrame ** result) + { + constexpr int kAlign = 32; + + if (pool_ == nullptr || pool_width_ != width || pool_height_ != height) { + av_buffer_pool_uninit(&pool_); + + const int size = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, width, height, kAlign); + + if (size < 0) { + return size; + } + + pool_ = av_buffer_pool_init(size, av_buffer_allocz); + pool_width_ = width; + pool_height_ = height; + + if (pool_ == nullptr) { + return AVERROR(ENOMEM); + } + } + + AVFrame * frame = av_frame_alloc(); + + if (frame == nullptr) { + return AVERROR(ENOMEM); + } + + // The buffer goes back to the pool when the last reference to the + // frame is dropped, so the memory of a picture is reused for the + // next ones rather than allocated, and faulted in, each time. + frame->buf[0] = av_buffer_pool_get(pool_); + + if (frame->buf[0] == nullptr) { + av_frame_free(&frame); + + return AVERROR(ENOMEM); + } + + frame->format = AV_PIX_FMT_YUV420P; + frame->width = width; + frame->height = height; + + int error = av_image_fill_arrays(frame->data, frame->linesize, frame->buf[0]->data, + AV_PIX_FMT_YUV420P, width, height, kAlign); + + if (error < 0) { + av_frame_free(&frame); + + return error; + } + + *result = frame; + + return 0; + } + int VideoDecoder::GetWidth() const { return codec_context_ != nullptr ? codec_context_->width : 0; From ffc55dda3248b104fadcc4c648b5456dd6b1938d Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 4 Oct 2026 18:02:30 +0200 Subject: [PATCH 2/2] perf: decode through Direct3D 12 first in the media player on Windows FFmpeg 8.1 has Direct3D 12 hwaccels for H.264 and VP9. The Windows build now enables them (h264_d3d12va, vp9_d3d12va), and the player tries a Direct3D 12 device before the Direct3D 11 one it used. A machine without Windows 10 2004 or a driver that decodes through Direct3D 12 has no such device, and goes on with Direct3D 11 as before. The pictures are the same: every test of the hardware path compares them with software decoding. Reading a picture back is what costs processor time on this path, and Direct3D 12 does it for about half of what Direct3D 11 does. Process CPU per frame in the player on an NVIDIA GeForce RTX 3080, median of four runs of steady playback, one machine and synthetic media: software Direct3D 11 Direct3D 12 1080p H.264 3.6 ms 4.2 ms 2.1 ms 4K H.264 6.6 ms 12.4 ms 5.4 ms Direct3D 11 used more processor time than software at 4K, Direct3D 12 less. The component list changed, so the next build rebuilds FFmpeg on Windows. --- docs/guide/media/media-files.md | 4 ++-- .../main/cpp/dependencies/ffmpeg/CMakeLists.txt | 16 ++++++++++++++-- .../src/main/cpp/src/media/VideoDecoder.cpp | 2 +- .../onvoid/webrtc/media/player/MediaPlayer.java | 5 +++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md index 085535240..a0dd35fd5 100644 --- a/docs/guide/media/media-files.md +++ b/docs/guide/media/media-files.md @@ -222,10 +222,10 @@ boolean hardware = source.getPlayer().isHardwareDecoding(); The pictures are the same ones software decoding gives; in the tests, every frame of H.264 and VP9 media is compared with its software counterpart. Hardware decoding is off unless asked for. -- **Platforms:** macOS through VideoToolbox, Windows through Direct3D 11, and Linux through NVDEC on NVIDIA GPUs (x86-64 and ARM64; the driver provides `libcuda.so.1` and `libnvcuvid.so.1`, which are loaded when a player asks for hardware). The flag is accepted everywhere; where the platform has no hardware decoder, or the FFmpeg build has none for it (32-bit ARM, VA-API on Intel and AMD GPUs), the video is decoded in software, and `isHardwareDecoding()` says so. +- **Platforms:** macOS through VideoToolbox, Windows through Direct3D 12 (Windows 10 version 2004 or later, with a driver that decodes video through it) and otherwise Direct3D 11, and Linux through NVDEC on NVIDIA GPUs (x86-64 and ARM64; the driver provides `libcuda.so.1` and `libnvcuvid.so.1`, which are loaded when a player asks for hardware). The flag is accepted everywhere; where the platform has no hardware decoder, or the FFmpeg build has none for it (32-bit ARM, VA-API on Intel and AMD GPUs), the video is decoded in software, and `isHardwareDecoding()` says so. - **Codecs:** H.264 and VP9. VP8, MPEG-4, MJPEG and H.265/HEVC are decoded in software. - **Fallback:** a stream the hardware does not take, such as a profile it cannot decode, is decoded in software without the player noticing more than `isHardwareDecoding()` turning `false`. If the hardware fails after it has produced a picture, decoding goes on in software from the next key frame. -- **What it saves** is processor time, not the copy: a decoded picture is read back from the media engine into system memory, and converted to I420 as WebRTC wants it. On an Apple M2, a 1080p H.264 stream took 11.6 ms of processor time per frame in software and 1.4 ms in hardware, and a 4K stream 35.7 ms against 4.2 ms, about seven to eight times less. Per frame, software on all cores is faster on the clock, which does not matter at playback speed. These are figures for one machine and for synthetic media, not a promise. +- **What it saves** is processor time, not the copy: a decoded picture is read back from the media engine into system memory, and converted to I420 as WebRTC wants it. On an Apple M2, a 1080p H.264 stream took 11.6 ms of processor time per frame in software and 1.4 ms in hardware, and a 4K stream 35.7 ms against 4.2 ms, about seven to eight times less. Per frame, software on all cores is faster on the clock, which does not matter at playback speed. Windows gains less: on an NVIDIA GeForce RTX 3080 through Direct3D 12, a 1080p H.264 stream took about 2.1 ms of processor time per frame against 3.6 ms in software, and a 4K stream 5.4 ms against 6.6 ms. Direct3D 11 reads the picture back more slowly and took more than software at 4K (12.4 ms), which is why Direct3D 12 comes first. These are figures for one machine each and for synthetic media, not a promise; try it on your own streams before relying on it. ## Playing a Live Stream diff --git a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt index b39f3b357..03dd875b6 100644 --- a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt +++ b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt @@ -132,8 +132,9 @@ if(APPLE) --enable-hwaccel=vp9_videotoolbox ) elseif(WIN32) - # Direct3D 11 only. DXVA2, on Direct3D 9, would add nothing on a GPU of - # the last decade, which decodes H.264 and VP9 through Direct3D 11. + # Direct3D 11, and 12 further down. DXVA2, on Direct3D 9, would add + # nothing on a GPU of the last decade, which decodes H.264 and VP9 + # through Direct3D 11. # FFmpeg loads d3d11.dll and dxgi.dll when a device is created, so a # machine without a GPU decoder loads the libraries all the same. # The decoders offer the D3D11 pixel format, which the *_d3d11va2 @@ -146,6 +147,17 @@ elseif(WIN32) --enable-hwaccel=h264_d3d11va2 --enable-hwaccel=vp9_d3d11va2 ) + # Direct3D 12 as well, which the player tries first: reading a picture + # back costs about half the processor time it does through Direct3D 11 at + # 4K. It needs Windows 10 2004 or later and a driver with Direct3D 12 + # video decoding; a machine without them has a Direct3D 11 decoder to + # fall back to, as it always had. Like Direct3D 11 it is loaded when a + # device is created. + list(APPEND FFMPEG_COMPONENTS + --enable-d3d12va + --enable-hwaccel=h264_d3d12va + --enable-hwaccel=vp9_d3d12va + ) elseif(LINUX AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|aarch64|arm64|ARM64)$") # NVDEC, on NVIDIA GPUs. FFmpeg loads the CUDA driver and libnvcuvid with # dlopen when a device is created, so the libraries load on a machine with diff --git a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp index 142f236f2..120976ae1 100644 --- a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -41,7 +41,7 @@ namespace #if defined(__APPLE__) return { AV_HWDEVICE_TYPE_VIDEOTOOLBOX }; #elif defined(_WIN32) - return { AV_HWDEVICE_TYPE_D3D11VA }; + return { AV_HWDEVICE_TYPE_D3D12VA, AV_HWDEVICE_TYPE_D3D11VA }; #else return { AV_HWDEVICE_TYPE_CUDA }; #endif diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java index 3cb298c2b..6a74f8910 100644 --- a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java @@ -106,8 +106,9 @@ public MediaPlayer(MediaReader reader, CustomVideoSource videoSource, *

* With {@code hardwareDecoding} the player decodes H.264 and VP9 video on * the media engine or GPU of the machine where there is one the platform - * offers (VideoToolbox on macOS, Direct3D 11 on Windows, NVDEC on NVIDIA - * GPUs on Linux), which takes a fraction of the processor time. The + * offers (VideoToolbox on macOS, Direct3D 12 or 11 on Windows, NVDEC on + * NVIDIA GPUs on Linux), which takes less processor time, by an amount + * that depends on the platform and the stream. The * pictures are the same as software decoding gives. A stream the * hardware does not take, or a codec it has no decoder for, is decoded in * software; {@link #isHardwareDecoding()} tells which it is.