diff --git a/.github/actions/prepare-linux/action.yml b/.github/actions/prepare-linux/action.yml index 4a36699b3..6d11aba6d 100644 --- a/.github/actions/prepare-linux/action.yml +++ b/.github/actions/prepare-linux/action.yml @@ -31,8 +31,9 @@ runs: - name: Install required packages run: | sudo apt-get update -q - # nasm is what FFmpeg assembles its x86 code with. - sudo apt-get install -y -q binutils cmake git locales lsb-release nasm ninja-build pipewire pipewire-pulse pkg-config python3 python3-setuptools rsync unzip wget xz-utils + # nasm is what FFmpeg assembles its x86 code with, and meson builds + # dav1d, the software decoder of AV1 in the media module. + sudo apt-get install -y -q binutils cmake git locales lsb-release meson nasm ninja-build pipewire pipewire-pulse pkg-config python3 python3-setuptools rsync unzip wget xz-utils shell: bash # Chromium Clang to be used with the clang toolchain file. The JNI code diff --git a/.github/actions/prepare-macos/action.yml b/.github/actions/prepare-macos/action.yml index cc07285e7..2d25f1a0c 100644 --- a/.github/actions/prepare-macos/action.yml +++ b/.github/actions/prepare-macos/action.yml @@ -16,6 +16,9 @@ runs: # nasm is what FFmpeg assembles its x86 code with, which the Intel # build needs even though the runner itself is Apple Silicon. brew install nasm + # Meson and Ninja build dav1d, the software decoder of AV1 in the media + # module, and FFmpeg's configure finds it with pkgconf. + brew install meson ninja pkgconf shell: bash - name: Select Xcode version diff --git a/.github/actions/prepare-windows/action.yml b/.github/actions/prepare-windows/action.yml index 98e9c1f70..36bb5b13a 100644 --- a/.github/actions/prepare-windows/action.yml +++ b/.github/actions/prepare-windows/action.yml @@ -8,6 +8,8 @@ runs: - name: Install required packages run: | choco install ninja + # Meson builds dav1d, the software decoder of AV1 in the media module. + python -m pip install meson shell: powershell - name: Install MSYS2 and the FFmpeg build tools diff --git a/.gitmodules b/.gitmodules index e237b00cb..caafec670 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = webrtc-java-media/third-party/ffmpeg url = https://github.com/FFmpeg/FFmpeg.git shallow = true +[submodule "webrtc-java-media/third-party/dav1d"] + path = webrtc-java-media/third-party/dav1d + url = https://code.videolan.org/videolan/dav1d.git + shallow = true diff --git a/NOTICE b/NOTICE index b726bb050..f891f8755 100644 --- a/NOTICE +++ b/NOTICE @@ -19,6 +19,20 @@ built from an unmodified FFmpeg release, without --enable-gpl and without jar carries the license text and a notice naming the FFmpeg version and where its source is, under META-INF/licenses/ffmpeg. +HEVC/H.265 + +Disclaimer regarding HEVC/H.265: +This project decodes HEVC/H.265 video through hardware acceleration only, using +the hardware decoders of the operating system or GPU (VideoToolbox, Direct3D +11/12, NVDEC). It does not include, bundle, or distribute any software-based +HEVC encoder (such as x265), and it never decodes HEVC in software. The FFmpeg +libraries in the webrtc-java-media platform jars that decode HEVC in hardware +(all but 32-bit ARM Linux) contain FFmpeg's HEVC decoder, because FFmpeg's +hardware decoding runs through it: it reads the stream's headers and passes the +coded data to the hardware decoder, which decodes the pictures. End-users and distributors are responsible for ensuring that the +underlying operating system or hardware platform holds a valid license for HEVC +playback and encoding. + DOCUMENTATION WebRTC 1.0: Real-time Communication Between Browsers diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md index 1c6fa501f..6a8032f2d 100644 --- a/docs/guide/media/media-files.md +++ b/docs/guide/media/media-files.md @@ -15,21 +15,21 @@ Decoding happens entirely in native code. Frames never travel through Java: the ## Adding the Module -The module is part of the normal build, and it builds FFmpeg from a submodule pinned to a release tag, so the submodule has to be present: +The module is part of the normal build, and it builds FFmpeg and dav1d (the software decoder of AV1) from submodules pinned to release tags, so the submodules have to be present: ```shell -git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg +git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg webrtc-java-media/third-party/dav1d mvn install ``` -Building FFmpeg needs `make` and `nasm`. On Windows they come from MSYS2: +Building FFmpeg needs `make` and `nasm`, and building dav1d needs [Meson](https://mesonbuild.com) (`pip install meson`) and Ninja. On Windows `make` and `nasm` come from MSYS2: ```shell winget install MSYS2.MSYS2 C:\msys64\usr\bin\bash -lc "pacman -S --needed make nasm diffutils pkgconf" ``` -Maven still runs from an ordinary shell; the build enters MSYS2 and the Visual Studio environment on its own. The first build compiles FFmpeg, which takes a while; later builds reuse the install directory. +Maven still runs from an ordinary shell; the build enters MSYS2 and the Visual Studio environment on its own. The first build compiles FFmpeg and dav1d, which takes a while; later builds reuse the install directory. Once installed, depend on it alongside `webrtc-java`. It takes two entries: one for the Java API, and one for the natives of the platform you are running on. @@ -206,7 +206,7 @@ It also keeps the native side of the media sources it feeds alive until it is cl ## Decoding in Hardware -Decoding H.264 and VP9 takes a good part of a processor at high resolutions, and for every stream of a camera wall. A player can decode them on the media engine or GPU of the machine instead. It does so when asked to, with a flag on `MediaFileSource` or `MediaPlayer`: +Decoding H.264, H.265/HEVC, VP9 and AV1 takes a good part of a processor at high resolutions, and for every stream of a camera wall. A player can decode them on the media engine or GPU of the machine instead. It does so when asked to, with a flag on `MediaFileSource` or `MediaPlayer`: ```java // A file whose video is decoded in hardware, where the platform can. @@ -220,11 +220,13 @@ MediaPlayer player = new MediaPlayer(new MediaReader(path), videoSource, null, t boolean hardware = source.getPlayer().isHardwareDecoding(); ``` -The pictures are the same ones software decoding gives; in the tests, every frame of H.264 and VP9 media is compared with its software counterpart. Hardware decoding is off unless asked for. +The pictures are the same ones software decoding gives; in the tests, every frame of media in each of these codecs is compared with its software counterpart (for HEVC, with checksums of the pictures FFmpeg's software decoder gave once). Hardware decoding is off unless asked for, except for H.265/HEVC. + +**HEVC is decoded in hardware only.** A player decodes it on the media engine or GPU whether or not the flag is set, and never in software: `isHardwareDecoding()` is always `true` for it. Where the machine has no hardware decoder for HEVC, creating the player fails with an `IOException`; where the hardware does not take a stream (a profile it cannot decode) or fails while playing, playback stops with an error. The fallback below does not apply to it. - **Platforms:** macOS through VideoToolbox, Windows through Direct3D 12 (Windows 10 version 2004 or later, with a driver that decodes video through it) and otherwise Direct3D 11, and Linux through NVDEC on NVIDIA GPUs (x86-64 and ARM64; the driver provides `libcuda.so.1` and `libnvcuvid.so.1`, which are loaded when a player asks for hardware). The flag is accepted everywhere; where the platform has no hardware decoder, or the FFmpeg build has none for it (32-bit ARM, VA-API on Intel and AMD GPUs), the video is decoded in software, and `isHardwareDecoding()` says so. -- **Codecs:** H.264 and VP9. VP8, MPEG-4, MJPEG and H.265/HEVC are decoded in software. -- **Fallback:** a stream the hardware does not take, such as a profile it cannot decode, is decoded in software without the player noticing more than `isHardwareDecoding()` turning `false`. If the hardware fails before its first picture, the packets it was sent are decoded again in software and none is lost; if it fails after one, decoding goes on in software from the next key frame. A damaged packet is not taken for a hardware failure after the first picture: as with software decoding, playback stops with an error, and the hardware keeps the stream. +- **Codecs:** H.264, H.265/HEVC, VP9 and AV1, where the GPU decodes them: a GPU that is a few years old may lack AV1, and on a Mac only an M3 or later has it. VP8, MPEG-4 and MJPEG are decoded in software, and so is AV1 where there is no GPU decoder for it (it is decoded by dav1d, which is built into the FFmpeg libraries). +- **Fallback** (every codec but HEVC): a stream the hardware does not take, such as a profile it cannot decode, is decoded in software without the player noticing more than `isHardwareDecoding()` turning `false`. If the hardware fails before its first picture, the packets it was sent are decoded again in software and none is lost; if it fails after one, decoding goes on in software from the next key frame. A damaged packet is not taken for a hardware failure after the first picture: as with software decoding, playback stops with an error, and the hardware keeps the stream. - **What it saves** is processor time, not the copy: a decoded picture is read back from the media engine into system memory, and converted to I420 as WebRTC wants it. On an Apple M2, a 1080p H.264 stream took 11.6 ms of processor time per frame in software and 1.4 ms in hardware, and a 4K stream 35.7 ms against 4.2 ms, about seven to eight times less. Per frame, software on all cores is faster on the clock, which does not matter at playback speed. Windows gains less: on an NVIDIA GeForce RTX 3080 through Direct3D 12, a 1080p H.264 stream took about 2.1 ms of processor time per frame against 3.6 ms in software, and a 4K stream 5.4 ms against 6.6 ms. Direct3D 11 reads the picture back more slowly and took more than software at 4K (12.4 ms), which is why Direct3D 12 comes first. These are figures for one machine each and for synthetic media, not a promise; try it on your own streams before relying on it. ## Playing a Live Stream @@ -269,7 +271,7 @@ The FFmpeg build is deliberately small, and carries only what this module plays: | | | | --- | --- | | **Containers** | MP4 and MOV, Matroska and WebM, AVI, MPEG-TS, FLV, WAV, MP3, Ogg, FLAC, AAC | -| **Video** | H.264, H.265/HEVC, VP8, VP9, MPEG-4 (including Xvid and DivX), Microsoft MPEG-4 v1 to v3, MJPEG | +| **Video** | H.264, H.265/HEVC (hardware only), VP8, VP9, AV1, MPEG-4 (including Xvid and DivX), Microsoft MPEG-4 v1 to v3, MJPEG | | **Audio** | AAC, MP3, MP2, AC-3, Opus, Vorbis, FLAC, PCM, MS and IMA ADPCM | Audio of any rate or layout is resampled to what WebRTC takes, which is 48 kHz 16-bit PCM in mono or stereo. Video that decodes to I420 — almost all 8-bit H.264, VP8, VP9 and MPEG-4 — reaches the encoder without being copied; anything else is converted first. diff --git a/docs/guide/media/media-recording.md b/docs/guide/media/media-recording.md index db3c23c95..fbb543d7d 100644 --- a/docs/guide/media/media-recording.md +++ b/docs/guide/media/media-recording.md @@ -48,6 +48,8 @@ The container follows the file name: A name FFmpeg does not know gets Matroska. A track whose codec the file cannot hold, VP8 in MP4 for example, is left out and reported as a warning, and the rest is recorded. +Matroska has no codec ID of its own for G.711, so it is stored as Windows audio (`A_MS/ACM`), the way FFmpeg itself stores it. Players built on FFmpeg, such as VLC and mpv, play it; some others do not. + ## Following a Recording A `MediaRecorderListener` hears when the file begins, what was left out, and whether writing failed: diff --git a/webrtc-java-media/README.md b/webrtc-java-media/README.md index 8ead425f0..4e656a290 100644 --- a/webrtc-java-media/README.md +++ b/webrtc-java-media/README.md @@ -62,7 +62,7 @@ This module is part of the normal build, and it builds FFmpeg from a submodule, has to be present: ```shell -git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg +git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg webrtc-java-media/third-party/dav1d mvn install ``` @@ -77,5 +77,9 @@ winget install MSYS2.MSYS2 C:\msys64\usr\bin\bash -lc "pacman -S --needed make nasm diffutils pkgconf" ``` +dav1d, the software decoder of AV1, is built first and linked into FFmpeg. Building it needs Meson +and Ninja (`pip install meson`, with `meson` on the `PATH`; `ninja` is there for the WebRTC build +already). + Maven still runs from an ordinary shell. The build enters MSYS2 and the Visual Studio environment on its own, because FFmpeg's configure needs a POSIX shell that can also see `cl` and `link`. diff --git a/webrtc-java-media/src/main/cpp/CMakeLists.txt b/webrtc-java-media/src/main/cpp/CMakeLists.txt index c31524bd7..a91e1cfbe 100644 --- a/webrtc-java-media/src/main/cpp/CMakeLists.txt +++ b/webrtc-java-media/src/main/cpp/CMakeLists.txt @@ -171,6 +171,24 @@ add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD "${FFMPEG_NOTICE_FILE}" "${FFMPEG_LICENSES_DIR}/NOTICE.txt" COMMENT "Collecting the FFmpeg license and notice for the platform jar") +# dav1d, the software decoder of AV1, is linked into libavcodec, and its license +# asks for its text to go along with the binaries. +set(DAV1D_LICENSE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../../../third-party/dav1d/COPYING") + +if(NOT EXISTS "${DAV1D_LICENSE_FILE}") + message(FATAL_ERROR + "${DAV1D_LICENSE_FILE} is missing, and the platform jar must not ship dav1d " + "without it. Check out the dav1d submodule.") +endif() + +set(DAV1D_LICENSES_DIR "${NATIVES_DIR}/META-INF/licenses/dav1d") + +add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "${DAV1D_LICENSES_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${DAV1D_LICENSE_FILE}" "${DAV1D_LICENSES_DIR}/COPYING" + COMMENT "Collecting the dav1d license for the platform jar") + # The headers FFmpeg was built against for NVDEC have licenses of their own, # which go along as well. if(FFMPEG_NVDEC) diff --git a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt index 9f766860d..59e68b1f8 100644 --- a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt +++ b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt @@ -56,7 +56,13 @@ set(FFMPEG_COMPONENTS --enable-demuxer=aac # Video this module decodes. --enable-decoder=h264 - --enable-decoder=hevc + # No HEVC decoder of its own: HEVC is decoded on the hardware only. The + # HEVC hwaccels below run inside FFmpeg's hevc decoder, which parses the + # parameter sets and slice headers, keeps the reference pictures and hands + # each slice to the hardware, and each of them selects that decoder, so it + # comes into the build with them and only where there are any. A target + # without one (32-bit ARM Linux) has no HEVC decoder at all. VideoDecoder + # never lets it decode a picture in software. --enable-decoder=vp8 --enable-decoder=vp9 --enable-decoder=mpeg4 @@ -120,32 +126,48 @@ set(FFMPEG_COMPONENTS --enable-protocol=tcp ) -# Decoding on the media engine or GPU, which a player does when it is asked to. -# H.264 and VP9 only: the codecs the module plays that platform decoders have -# too, and ones it can fall back to software for. +# Decoding on the media engine or GPU, which a player does when it is asked to: +# H.264, HEVC, VP9 and AV1, the codecs the module plays that platform decoders +# have too. The native AV1 decoder of FFmpeg has no software path, it exists for +# the hardware, so AV1 has dav1d to fall back to, which is built beside FFmpeg +# and linked into libavcodec (see the dav1d build below). FFmpeg prefers it to +# its own decoder, and the player picks the other one for the hardware. +list(APPEND FFMPEG_COMPONENTS + --enable-decoder=av1 + --enable-libdav1d + --enable-decoder=libdav1d +) + if(APPLE) # VideoToolbox is part of macOS, so nothing is added to what the libraries - # need to load. + # need to load. AV1 is decoded by the media engine of an M3 or later only, + # and a Mac without it takes the software decoder. list(APPEND FFMPEG_COMPONENTS --enable-videotoolbox --enable-hwaccel=h264_videotoolbox + --enable-hwaccel=hevc_videotoolbox --enable-hwaccel=vp9_videotoolbox + --enable-hwaccel=av1_videotoolbox ) elseif(WIN32) # Direct3D 11, and 12 further down. DXVA2, on Direct3D 9, would add - # nothing on a GPU of the last decade, which decodes H.264 and VP9 - # through Direct3D 11. + # nothing on a GPU of the last decade, which decodes H.264, HEVC, VP9 and + # AV1 through Direct3D 11. # FFmpeg loads d3d11.dll and dxgi.dll when a device is created, so a # machine without a GPU decoder loads the libraries all the same. # The decoders offer the D3D11 pixel format, which the *_d3d11va2 # hwaccels decode into, only when the older *_d3d11va ones are enabled as - # well (h264_slice.c, vp9.c), so both are needed. + # well (h264_slice.c, vp9.c, hevcdec.c, av1dec.c), so both are needed. list(APPEND FFMPEG_COMPONENTS --enable-d3d11va --enable-hwaccel=h264_d3d11va + --enable-hwaccel=hevc_d3d11va --enable-hwaccel=vp9_d3d11va + --enable-hwaccel=av1_d3d11va --enable-hwaccel=h264_d3d11va2 + --enable-hwaccel=hevc_d3d11va2 --enable-hwaccel=vp9_d3d11va2 + --enable-hwaccel=av1_d3d11va2 ) # Direct3D 12 as well, which the player tries first: reading a picture # back costs about half the processor time it does through Direct3D 11 at @@ -156,7 +178,9 @@ elseif(WIN32) list(APPEND FFMPEG_COMPONENTS --enable-d3d12va --enable-hwaccel=h264_d3d12va + --enable-hwaccel=hevc_d3d12va --enable-hwaccel=vp9_d3d12va + --enable-hwaccel=av1_d3d12va ) elseif(LINUX AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|aarch64|arm64|ARM64)$") # NVDEC, on NVIDIA GPUs. FFmpeg loads the CUDA driver and libnvcuvid with @@ -170,7 +194,9 @@ elseif(LINUX AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64|aarch64|ar --enable-ffnvcodec --enable-nvdec --enable-hwaccel=h264_nvdec + --enable-hwaccel=hevc_nvdec --enable-hwaccel=vp9_nvdec + --enable-hwaccel=av1_nvdec ) endif() @@ -205,6 +231,49 @@ foreach(PATCH ${FFMPEG_PATCHES}) string(APPEND FFMPEG_COMPONENTS_TEXT "\npatch ${PATCH_NAME} ${PATCH_HASH}") endforeach() +# +# dav1d is the software decoder of AV1, built from the third-party/dav1d +# submodule with Meson into a prefix of its own, as a static library, which +# FFmpeg then links into libavcodec. Its version and the options it is built +# with belong to the install like the components do, so that an install made +# with another version or other options is built again. +# +set(DAV1D_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../third-party/dav1d") +get_filename_component(DAV1D_SOURCE_DIR "${DAV1D_SOURCE_DIR}" ABSOLUTE) + +set(DAV1D_OPTIONS + --buildtype=release + --default-library=static + -Denable_tools=false + -Denable_tests=false + -Denable_examples=false + -Denable_docs=false +) + +if(WIN32) + # FFmpeg is compiled without a choice of C runtime, which for MSVC is the + # static one, so dav1d has to be as well. The dynamic one, which Meson + # takes for a release build, does not link into it. + list(APPEND DAV1D_OPTIONS -Db_vscrt=mt) +endif() + +set(DAV1D_VERSION "missing") + +if(EXISTS "${DAV1D_SOURCE_DIR}/meson.build") + file(STRINGS "${DAV1D_SOURCE_DIR}/meson.build" DAV1D_VERSION_LINE + REGEX "^[ ]+version: '[0-9]+\\.[0-9]+\\.[0-9]+'," LIMIT_COUNT 1) + + if(DAV1D_VERSION_LINE MATCHES "'([0-9]+\\.[0-9]+\\.[0-9]+)'") + set(DAV1D_VERSION "${CMAKE_MATCH_1}") + endif() +endif() + +string(REPLACE ";" " " DAV1D_OPTIONS_TEXT "${DAV1D_OPTIONS}") +string(APPEND FFMPEG_COMPONENTS_TEXT "\ndav1d ${DAV1D_VERSION} ${DAV1D_OPTIONS_TEXT}") + +# The parent names the version in the notice that goes into the platform jar. +set(DAV1D_VERSION "${DAV1D_VERSION}" PARENT_SCOPE) + # Applies the patches to the source. One that is applied already is left as it # is: a build that was cut short does not get to take them off. function(ffmpeg_apply_patches) @@ -359,7 +428,8 @@ else() # shell, so check for them there rather than with find_program. get_filename_component(MSYS2_BIN_DIR "${BASH_EXECUTABLE}" DIRECTORY) - foreach(REQUIRED_TOOL make nasm) + # pkgconf is how FFmpeg's configure finds dav1d. + foreach(REQUIRED_TOOL make nasm pkgconf) if(NOT EXISTS "${MSYS2_BIN_DIR}/${REQUIRED_TOOL}.exe") message(FATAL_ERROR "MSYS2 has no '${REQUIRED_TOOL}', which building FFmpeg needs. Run inside " @@ -455,6 +525,7 @@ else() --disable-avfilter --disable-everything --disable-autodetect + --pkg-config-flags=--static ${FFMPEG_COMPONENTS} ) @@ -591,6 +662,165 @@ else() list(APPEND FFMPEG_CONFIGURE_ARGS "--extra-ldflags=\"${FFMPEG_EXTRA_LDFLAGS}\"") endif() + # + # dav1d, built before FFmpeg, whose configure finds it through pkg-config. + # It is built for the same target as FFmpeg, from what this file already + # worked out about it: for the host as it is, with the compiler and flags + # in the environment, and for another target with a cross file. + # + if(NOT EXISTS "${DAV1D_SOURCE_DIR}/meson.build") + message(FATAL_ERROR + "The dav1d submodule is not checked out. Run:\n" + " git submodule update --init --depth 1 webrtc-java-media/third-party/dav1d") + endif() + + find_program(MESON_EXECUTABLE NAMES meson) + + if(NOT MESON_EXECUTABLE) + message(FATAL_ERROR + "No 'meson' found, which building dav1d needs. Install it, for example with:\n" + " pip install meson\n" + "and make sure it is on the PATH. Meson also needs Ninja, which building WebRTC " + "needs as well.") + endif() + + set(DAV1D_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/dav1d-build") + set(DAV1D_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/dav1d-install") + + file(REMOVE_RECURSE "${DAV1D_BUILD_DIR}" "${DAV1D_INSTALL_DIR}") + + set(DAV1D_SETUP_OPTIONS ${DAV1D_OPTIONS} --libdir=lib) + set(DAV1D_ENVIRONMENT "") + + if(FFMPEG_CROSS_COMPILING) + if(WIN32) + set(DAV1D_SYSTEM windows) + set(DAV1D_COMPILER cl) + elseif(APPLE) + set(DAV1D_SYSTEM darwin) + set(DAV1D_COMPILER cc) + else() + set(DAV1D_SYSTEM linux) + set(DAV1D_COMPILER "${CMAKE_C_COMPILER}") + endif() + + if(FFMPEG_ARCH STREQUAL "arm64") + set(DAV1D_CPU_FAMILY aarch64) + else() + set(DAV1D_CPU_FAMILY "${FFMPEG_ARCH}") + endif() + + # The flags FFmpeg is given go to dav1d as well, as the Meson options + # that hold the flags of the C compiler and linker. + set(DAV1D_C_ARGS "${FFMPEG_EXTRA_CFLAGS}") + set(DAV1D_C_LINK_ARGS "${FFMPEG_EXTRA_LDFLAGS}") + + if(CMAKE_SYSROOT) + string(APPEND DAV1D_C_ARGS " --sysroot=${CMAKE_SYSROOT}") + string(APPEND DAV1D_C_LINK_ARGS " --sysroot=${CMAKE_SYSROOT}") + endif() + + separate_arguments(DAV1D_C_ARGS UNIX_COMMAND "${DAV1D_C_ARGS}") + separate_arguments(DAV1D_C_LINK_ARGS UNIX_COMMAND "${DAV1D_C_LINK_ARGS}") + + # Meson reads these as lists of quoted strings. + set(DAV1D_C_ARGS_TEXT "") + foreach(ARG ${DAV1D_C_ARGS}) + list(APPEND DAV1D_C_ARGS_TEXT "'${ARG}'") + endforeach() + string(REPLACE ";" ", " DAV1D_C_ARGS_TEXT "${DAV1D_C_ARGS_TEXT}") + + set(DAV1D_C_LINK_ARGS_TEXT "") + foreach(ARG ${DAV1D_C_LINK_ARGS}) + list(APPEND DAV1D_C_LINK_ARGS_TEXT "'${ARG}'") + endforeach() + string(REPLACE ";" ", " DAV1D_C_LINK_ARGS_TEXT "${DAV1D_C_LINK_ARGS_TEXT}") + + set(DAV1D_BINARIES "c = ['${DAV1D_COMPILER}']") + + if(CMAKE_AR) + string(APPEND DAV1D_BINARIES "\nar = ['${CMAKE_AR}']") + endif() + + set(DAV1D_CROSS_FILE "${CMAKE_CURRENT_BINARY_DIR}/dav1d-cross.ini") + + file(WRITE "${DAV1D_CROSS_FILE}" +"# Generated by CMake. The target dav1d is built for. +[binaries] +${DAV1D_BINARIES} + +[host_machine] +system = '${DAV1D_SYSTEM}' +cpu_family = '${DAV1D_CPU_FAMILY}' +cpu = '${DAV1D_CPU_FAMILY}' +endian = 'little' + +[properties] +needs_exe_wrapper = true + +[built-in options] +c_args = [${DAV1D_C_ARGS_TEXT}] +c_link_args = [${DAV1D_C_LINK_ARGS_TEXT}] +") + + list(APPEND DAV1D_SETUP_OPTIONS --cross-file "${DAV1D_CROSS_FILE}") + + # No assembler for a foreign architecture, as with FFmpeg above. + if(FFMPEG_ARCH STREQUAL "aarch64" OR FFMPEG_ARCH STREQUAL "arm64" OR FFMPEG_ARCH STREQUAL "arm") + list(APPEND DAV1D_SETUP_OPTIONS -Denable_asm=false) + endif() + elseif(NOT WIN32) + # On Windows the compiler is the one vcvars puts first on the PATH. + if(NOT APPLE AND CMAKE_C_COMPILER) + string(APPEND DAV1D_ENVIRONMENT "export CC=\"${CMAKE_C_COMPILER}\"\n") + endif() + if(NOT FFMPEG_EXTRA_CFLAGS STREQUAL "") + string(APPEND DAV1D_ENVIRONMENT "export CFLAGS=\"${FFMPEG_EXTRA_CFLAGS}\"\n") + endif() + if(NOT FFMPEG_EXTRA_LDFLAGS STREQUAL "") + string(APPEND DAV1D_ENVIRONMENT "export LDFLAGS=\"${FFMPEG_EXTRA_LDFLAGS}\"\n") + endif() + endif() + + # Meson finds the compiler by its path inside the Xcode toolchain, which + # skips the /usr/bin/cc shim that points SDKROOT at an SDK, and without one + # there is no libSystem to link against: "cannot compile programs". So the + # SDK is named here, for Meson only, as FFmpeg's own configure gets it from + # the shim. + if(APPLE) + string(APPEND DAV1D_ENVIRONMENT "export SDKROOT=\"$(xcrun --sdk macosx --show-sdk-path)\"\n") + endif() + + string(REPLACE ";" " " DAV1D_SETUP_STR "${DAV1D_SETUP_OPTIONS}") + + to_shell_path("${DAV1D_INSTALL_DIR}" DAV1D_INSTALL_DIR_SH) + to_shell_path("${MESON_EXECUTABLE}" MESON_EXECUTABLE_SH) + + # Meson, which is a native program even on Windows, is given the paths in + # the form CMake has them in. The static library is installed as libdav1d.a + # whatever the compiler, and the linker of MSVC is told to look for + # dav1d.lib. + set(DAV1D_BUILD_STEPS +"${DAV1D_ENVIRONMENT}\"${MESON_EXECUTABLE_SH}\" setup \"${DAV1D_BUILD_DIR}\" \"${DAV1D_SOURCE_DIR}\" --prefix=\"${DAV1D_INSTALL_DIR}\" ${DAV1D_SETUP_STR} +\"${MESON_EXECUTABLE_SH}\" install -C \"${DAV1D_BUILD_DIR}\" +unset CC CFLAGS LDFLAGS SDKROOT +") + + if(WIN32) + string(APPEND DAV1D_BUILD_STEPS +"cp \"${DAV1D_INSTALL_DIR_SH}/lib/libdav1d.a\" \"${DAV1D_INSTALL_DIR_SH}/lib/dav1d.lib\" +") + endif() + + # FFmpeg finds dav1d through pkg-config, which --pkg-config-flags=--static + # above has it read what a static library links against with as well. Only + # our directory is searched: a sysroot or library path of the host would put + # the include directory somewhere it is not. + string(APPEND DAV1D_BUILD_STEPS +"export PKG_CONFIG_PATH=\"${DAV1D_INSTALL_DIR_SH}/lib/pkgconfig:$PKG_CONFIG_PATH\" +unset PKG_CONFIG_SYSROOT_DIR PKG_CONFIG_LIBDIR +") + # Build out of tree so the submodule working tree stays clean. The patches # are put on it for the length of the build only. set(FFMPEG_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/ffmpeg-build") @@ -674,7 +904,7 @@ unset PKG_CONFIG_SYSROOT_DIR PKG_CONFIG_LIBDIR file(WRITE "${FFMPEG_BUILD_SCRIPT}" "# Generated by CMake. Builds FFmpeg ${FFMPEG_VERSION} for webrtc-java-media. set -e -${FFMPEG_SHELL_PREAMBLE}cd \"${FFMPEG_BUILD_DIR_SH}\" +${FFMPEG_SHELL_PREAMBLE}${DAV1D_BUILD_STEPS}cd \"${FFMPEG_BUILD_DIR_SH}\" # FFmpeg names its version after the git tag it finds, and a shallow clone of # the submodule has none, so it would name itself after a commit hash. Handing # make the release the source says it is names it the same way however the diff --git a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/FFmpeg-NOTICE.txt.in b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/FFmpeg-NOTICE.txt.in index b675cd170..d0c0522cc 100644 --- a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/FFmpeg-NOTICE.txt.in +++ b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/FFmpeg-NOTICE.txt.in @@ -14,6 +14,18 @@ separate shared libraries, which you may replace with your own build of a compatible FFmpeg version. +dav1d +----- + +libavcodec contains dav1d @DAV1D_VERSION@ (https://code.videolan.org/videolan/dav1d), +the AV1 decoder of the VideoLAN project, linked in statically. dav1d is +licensed under the BSD 2-Clause license, whose text is in +META-INF/licenses/dav1d/COPYING of this jar. It is built from the release tag +@DAV1D_VERSION@, without modifications: + + https://code.videolan.org/videolan/dav1d/-/tree/@DAV1D_VERSION@ + + Source code ----------- @@ -37,3 +49,19 @@ Patents Some of the formats FFmpeg decodes, such as H.264, H.265/HEVC and AAC, may be covered by patents in some countries. Whether your use of them needs a patent license is for you to determine. + + +HEVC/H.265 +---------- + +Disclaimer regarding HEVC/H.265: +This project decodes HEVC/H.265 video through hardware acceleration only, using +the hardware decoders of the operating system or GPU (VideoToolbox, Direct3D +11/12, NVDEC). It does not include, bundle, or distribute any software-based +HEVC encoder (such as x265), and it never decodes HEVC in software. The FFmpeg +libraries in the webrtc-java-media platform jars that decode HEVC in hardware +(all but 32-bit ARM Linux) contain FFmpeg's HEVC decoder, because FFmpeg's +hardware decoding runs through it: it reads the stream's headers and passes the +coded data to the hardware decoder, which decodes the pictures. End-users and distributors are responsible for ensuring that the +underlying operating system or hardware platform holds a valid license for HEVC +playback and encoding. diff --git a/webrtc-java-media/src/main/cpp/include/JNI_HardwareFault.h b/webrtc-java-media/src/main/cpp/include/JNI_HardwareFault.h index 7a7f3aba0..0f887962d 100644 --- a/webrtc-java-media/src/main/cpp/include/JNI_HardwareFault.h +++ b/webrtc-java-media/src/main/cpp/include/JNI_HardwareFault.h @@ -38,6 +38,14 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_HardwareFault_failDrain (JNIEnv *, jclass); + /* + * Class: dev_onvoid_webrtc_media_player_HardwareFault + * Method: hideDevices + * Signature: ()V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_HardwareFault_hideDevices + (JNIEnv *, jclass); + /* * Class: dev_onvoid_webrtc_media_player_HardwareFault * Method: disarm diff --git a/webrtc-java-media/src/main/cpp/include/media/HardwareFault.h b/webrtc-java-media/src/main/cpp/include/media/HardwareFault.h index 85f0f17f5..54fb2cd05 100644 --- a/webrtc-java-media/src/main/cpp/include/media/HardwareFault.h +++ b/webrtc-java-media/src/main/cpp/include/media/HardwareFault.h @@ -38,6 +38,10 @@ namespace ffmpeg // to drain: the failure of a decoder that never delivered. static void FailDrain(int error); + // Makes the platform look as if it had no hardware device to + // decode with, until disarmed: a machine without a GPU decoder. + static void HideDevices(); + // Forgets any fault, and the software decoder noted last. static void Disarm(); @@ -57,6 +61,9 @@ namespace ffmpeg // The error to fail the receive of a picture with once draining. static int DrainError(); + + // Whether the hardware devices are hidden. + static bool DevicesHidden(); }; } diff --git a/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h index 30d4c1dcd..606f2dc9b 100644 --- a/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h +++ b/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h @@ -39,11 +39,16 @@ namespace ffmpeg // with swscale first, which costs one copy and is the price of the format // rather than of this design. // - // Asked to, it decodes H.264 and VP9 on the media engine or GPU instead. + // Asked to, it decodes H.264, HEVC, VP9 and AV1 on the media engine or GPU + // instead. // The pictures that come out are the same; they are read back into system // memory and converted, so the gain is processor time, not the copy. A // stream the hardware cannot decode is decoded in software, without the // caller noticing more than IsHardware turning false. + // + // HEVC is the exception: it is decoded on the hardware only, whether that + // is asked for or not, and never in software. Where the hardware cannot + // decode it, opening or decoding fails. class VideoDecoder { public: @@ -56,6 +61,8 @@ namespace ffmpeg // Opens a decoder for the given stream, which must outlive this // decoder, in hardware if that is asked for and the platform has // a hardware decoder for the codec, and in software otherwise. + // HEVC goes to the hardware in any case, and without a hardware + // decoder for it the result is AVERROR_DECODER_NOT_FOUND. // Returns 0 or a negative AVERROR. int Open(const AVStream * stream, bool hardware = false); diff --git a/webrtc-java-media/src/main/cpp/src/JNI_HardwareFault.cpp b/webrtc-java-media/src/main/cpp/src/JNI_HardwareFault.cpp index 006ae3ac0..281db3a89 100644 --- a/webrtc-java-media/src/main/cpp/src/JNI_HardwareFault.cpp +++ b/webrtc-java-media/src/main/cpp/src/JNI_HardwareFault.cpp @@ -38,6 +38,12 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_HardwareFault_failDra ffmpeg::HardwareFault::FailDrain(AVERROR_EXTERNAL); } +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_HardwareFault_hideDevices +(JNIEnv * env, jclass caller) +{ + ffmpeg::HardwareFault::HideDevices(); +} + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_player_HardwareFault_disarm (JNIEnv * env, jclass caller) { diff --git a/webrtc-java-media/src/main/cpp/src/media/HardwareFault.cpp b/webrtc-java-media/src/main/cpp/src/media/HardwareFault.cpp index cecbfa608..b52234c55 100644 --- a/webrtc-java-media/src/main/cpp/src/media/HardwareFault.cpp +++ b/webrtc-java-media/src/main/cpp/src/media/HardwareFault.cpp @@ -31,6 +31,7 @@ namespace std::atomic error_code{ 0 }; std::atomic packets_left{ 0 }; std::atomic software_threads{ 0 }; + std::atomic devices_hidden{ false }; } namespace ffmpeg @@ -48,10 +49,16 @@ namespace ffmpeg mode.store(kDrain); } + void HardwareFault::HideDevices() + { + devices_hidden.store(true); + } + void HardwareFault::Disarm() { mode.store(kNone); software_threads.store(0); + devices_hidden.store(false); } void HardwareFault::NoteSoftwareDecoder(int threads) @@ -91,4 +98,9 @@ namespace ffmpeg return error_code.load(); } + + bool HardwareFault::DevicesHidden() + { + return devices_hidden.load(); + } } diff --git a/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp b/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp index bbb3a4a7e..388beb705 100644 --- a/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp +++ b/webrtc-java-media/src/main/cpp/src/media/RecorderTrack.cpp @@ -77,11 +77,12 @@ namespace ffmpeg // Whether the container holds the codec. The codec tags say most of // it, but the MP4 family refuses some codecs only when the header is // written, which would fail the whole file rather than leave out one - // track. + // track, and Matroska holds more than its tags say. bool CanHold(const AVOutputFormat * format, AVCodecID codec_id) { const bool mp4 = std::strcmp(format->name, "mp4") == 0; const bool mov = std::strcmp(format->name, "mov") == 0; + const bool matroska = std::strcmp(format->name, "matroska") == 0; if ((mp4 || mov) && codec_id == AV_CODEC_ID_VP8) { return false; @@ -89,6 +90,14 @@ namespace ffmpeg if (mov && (codec_id == AV_CODEC_ID_VP9 || codec_id == AV_CODEC_ID_AV1)) { return false; } + // Matroska has no codec ID of its own for G.711, so its tags say + // no. Its muxer writes it all the same, as it does any audio with + // a WAV format tag: in the mode kept for Windows audio codecs + // (A_MS/ACM, the WAVEFORMATEX in the codec private data), which + // is what "ffmpeg -c copy" makes of it too. WebM has no such mode. + if (matroska && (codec_id == AV_CODEC_ID_PCM_MULAW || codec_id == AV_CODEC_ID_PCM_ALAW)) { + return true; + } return avformat_query_codec(format, codec_id, FF_COMPLIANCE_NORMAL) == 1; } diff --git a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp index b91084fc5..72e71aa5c 100644 --- a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp +++ b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -48,6 +48,41 @@ namespace #endif } + // Whether the codec is decoded on the hardware only, never in software: + // HEVC. FFmpeg's decoder of it is in the build all the same, because the + // hardware decoders run inside it (it reads the headers and hands the + // slices to the hardware), but it decodes no picture itself. Where the + // hardware cannot, opening or decoding the stream fails. + bool IsHardwareOnly(AVCodecID id) + { + return id == AV_CODEC_ID_HEVC; + } + + // The decoder to open for a codec. Software gets the one FFmpeg prefers. + // Hardware gets it too, unless it has no hardware configuration: where + // another decoder of the codec stands in front of the one that decodes on + // the hardware, as libdav1d does for the AV1 decoder of FFmpeg itself (which + // has no software path), the hardware has to be given the latter. + const AVCodec * FindDecoder(AVCodecID id, bool hardware) + { + const AVCodec * preferred = avcodec_find_decoder(id); + + if (!hardware || preferred == nullptr || avcodec_get_hw_config(preferred, 0) != nullptr) { + return preferred; + } + + void * iterator = nullptr; + + while (const AVCodec * candidate = av_codec_iterate(&iterator)) { + if (candidate->id == id && av_codec_is_decoder(candidate) + && avcodec_get_hw_config(candidate, 0) != nullptr) { + return candidate; + } + } + + return preferred; + } + bool IsFailure(int result) { return result < 0 && result != AVERROR(EAGAIN) && result != AVERROR_EOF; @@ -80,9 +115,27 @@ namespace ffmpeg stream_ = stream; + // A codec decoded on the hardware only goes there whether that was + // asked for or not. + const bool hardware_only = IsHardwareOnly(stream->codecpar->codec_id); + + if (hardware_only) { + hardware = true; + } + int result = OpenContext(hardware); - if (result < 0 && hardware) { + if (result < 0 && hardware_only) { + av_log(nullptr, AV_LOG_ERROR, "No hardware decoder for %s (%s), which is not decoded in software\n", + avcodec_get_name(stream->codecpar->codec_id), ErrorString(result).c_str()); + + if (result == AVERROR(ENOSYS)) { + // No hardware device or configuration for it: as far as this + // machine goes, there is no decoder. + result = AVERROR_DECODER_NOT_FOUND; + } + } + else if (result < 0 && hardware) { // Verbose: a codec with no hardware decoder is nothing to report, // and IsHardware tells a caller that wants to know. av_log(nullptr, AV_LOG_VERBOSE, "No hardware video decoder (%s), decoding in software\n", @@ -118,7 +171,11 @@ namespace ffmpeg { software_selected_ = false; - const AVCodec * codec = avcodec_find_decoder(stream_->codecpar->codec_id); + if (!hardware && IsHardwareOnly(stream_->codecpar->codec_id)) { + return AVERROR_DECODER_NOT_FOUND; + } + + const AVCodec * codec = FindDecoder(stream_->codecpar->codec_id, hardware); if (codec == nullptr) { return AVERROR_DECODER_NOT_FOUND; @@ -179,6 +236,10 @@ namespace ffmpeg bool VideoDecoder::SetUpHardware(const AVCodec * codec) { + if (HardwareFault::DevicesHidden()) { + return false; + } + for (AVHWDeviceType type : DeviceTypes()) { for (int index = 0;; index++) { const AVCodecHWConfig * config = avcodec_get_hw_config(codec, index); @@ -221,7 +282,13 @@ namespace ffmpeg } // The decoder does not offer the hardware format for this stream: - // a profile or a size the hardware does not take. Software it is, and + // a profile or a size the hardware does not take. + if (IsHardwareOnly(context->codec_id)) { + // No software format either, so the stream fails to decode. + return AV_PIX_FMT_NONE; + } + + // Software it is, and // the context made for hardware, which has one thread, is replaced by // one made for software. decoder->software_selected_ = true; @@ -491,6 +558,17 @@ namespace ffmpeg int VideoDecoder::FallBackToSoftware(int error, const AVPacket * current) { + if (IsHardwareOnly(stream_->codecpar->codec_id)) { + av_log(nullptr, AV_LOG_ERROR, "Hardware video decoding of %s failed (%s), and it is not decoded in software\n", + avcodec_get_name(stream_->codecpar->codec_id), ErrorString(error).c_str()); + + probation_ = false; + + DropKeptPackets(); + + return error < 0 ? error : AVERROR_DECODER_NOT_FOUND; + } + if (error < 0) { av_log(nullptr, AV_LOG_WARNING, "Hardware video decoding failed (%s), decoding in software\n", ErrorString(error).c_str()); diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java index 88a06d3f6..c7ec2df53 100644 --- a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaFileSource.java @@ -87,6 +87,7 @@ public MediaFileSource(Path path) throws IOException { * * @param path The path of the file to play. * @param hardwareDecoding Whether to decode video in hardware if possible. + * HEVC is decoded in hardware regardless. * * @throws IOException if the source cannot be opened or decoded. */ @@ -112,6 +113,7 @@ public MediaFileSource(String source) throws IOException { * * @param source The path or {@code rtsp://} URL of the source to play. * @param hardwareDecoding Whether to decode video in hardware if possible. + * HEVC is decoded in hardware regardless. * * @throws IOException if the source cannot be opened or decoded. */ @@ -145,6 +147,7 @@ public MediaFileSource(String source, Duration timeout) throws IOException { * wait on the source. {@link Duration#ZERO} waits * as long as it takes. * @param hardwareDecoding Whether to decode video in hardware if possible. + * HEVC is decoded in hardware regardless. * * @throws IOException if the source cannot be opened in time, or decoded. */ diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java index 6a74f8910..930609762 100644 --- a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/player/MediaPlayer.java @@ -104,21 +104,30 @@ public MediaPlayer(MediaReader reader, CustomVideoSource videoSource, * media sources. At least one of them has to be present; media of a kind * with no source is decoded and dropped. *

- * With {@code hardwareDecoding} the player decodes H.264 and VP9 video on - * the media engine or GPU of the machine where there is one the platform + * With {@code hardwareDecoding} the player decodes H.264, H.265/HEVC, VP9 + * and AV1 video on the media engine or GPU of the machine where there is one the platform * offers (VideoToolbox on macOS, Direct3D 12 or 11 on Windows, NVDEC on * NVIDIA GPUs on Linux), which takes less processor time, by an amount * that depends on the platform and the stream. The * pictures are the same as software decoding gives. A stream the * hardware does not take, or a codec it has no decoder for, is decoded in * software; {@link #isHardwareDecoding()} tells which it is. + *

+ * H.265/HEVC is decoded in hardware only, with or without + * {@code hardwareDecoding}, and never in software. Without a hardware + * decoder for it the player cannot be created, and if the hardware fails + * while playing, or does not take the stream, playback stops with an + * error. * * @param reader The source to play, which this player takes over. * @param videoSource Where video goes, or {@code null} for none. * @param audioSource Where audio goes, or {@code null} for none. * @param hardwareDecoding Whether to decode video in hardware if possible. + * HEVC is decoded in hardware regardless. * - * @throws IOException if the source cannot be decoded. + * @throws IOException if the source cannot be decoded, + * including HEVC on a machine without a + * hardware decoder for it. * @throws IllegalArgumentException if both sources are {@code null}. */ public MediaPlayer(MediaReader reader, CustomVideoSource videoSource, @@ -215,7 +224,8 @@ public long getPositionUs() { * less than was asked for: the platform may have no hardware decoder for * the codec, or the hardware may not take the stream, in which case the * player decodes in software. It can turn {@code false} while playing, if - * the hardware decoder fails, and never turns {@code true} again. + * the hardware decoder fails, and never turns {@code true} again. For + * HEVC, which is decoded in hardware only, it is always {@code true}. * * @return {@code true} if video is decoded in hardware. */ diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java index 9dd527738..670f0d56f 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareDecodingTest.java @@ -20,10 +20,14 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -55,12 +59,15 @@ /** * Tests decoding video in hardware: that a player asked to decode in hardware * sends the very pictures software decoding does, and that it falls back to - * software where the hardware has nothing to offer. + * software where the hardware has nothing to offer. HEVC, which is decoded in + * hardware only, is compared with reference checksums instead, and must never + * reach a software decoder. *

* A machine without a hardware decoder for the codec skips the tests that need * one, as CI runners do. Set the system property {@code * webrtc.test.hardwareDecoding} to {@code true} on a machine that has one, to - * make those tests fail instead. + * make those tests fail instead, and {@code webrtc.test.hardwareAv1Decoding} + * for a GPU that decodes AV1. *

* The assets are 320x240 at 15 fps, two seconds, with a key frame every second: * {@code media-test-h264.mkv} is H.264 High made by the VideoToolbox encoder, @@ -76,6 +83,9 @@ class HardwareDecodingTest { private static final boolean HARDWARE_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareDecoding"); + /** AV1 needs a newer GPU than the other codecs, so it has a property of its own. */ + private static final boolean HARDWARE_AV1_REQUIRED = Boolean.getBoolean("webrtc.test.hardwareAv1Decoding"); + private static final String H264 = "/media-test-h264.mkv"; private static final String VP9 = "/media-test-vp9.webm"; @@ -107,6 +117,34 @@ class HardwareDecodingTest { private static final String H264_444 = "/media-test-h264-444.mkv"; + /** + * HEVC (Main) made with libx265, 320x240 at 15 fps, two seconds with a key + * frame every second, and the same in 1080p, one second with a key frame + * every eight frames. Both are the moving gradient with noise of the other + * assets, for example: + * 

+	 * ffmpeg -f lavfi -i "gradients=s=1920x1080:r=15:d=1:speed=0.02,noise=alls=4:allf=t,format=yuv420p" \
+	 *     -c:v libx265 -preset fast -b:v 3M -maxrate 4M -bufsize 4M \
+	 *     -x265-params keyint=8:min-keyint=8:log-level=error -an media-test-hevc-1080p.mkv
+	 *
+ * {@code media-test-hevc.luma.txt} and {@code media-test-hevc-1080p.luma.txt} + * hold the luma checksums of their pictures, one a line, as FFmpeg 8.1's + * software decoder gave them before this module stopped decoding HEVC in + * software. + */ + private static final String HEVC = "/media-test-hevc.mkv"; + + private static final String HEVC_1080P = "/media-test-hevc-1080p.mkv"; + + /** + * AV1 (Main) made with SVT-AV1, in the same sizes as the HEVC assets and + * the same way, with {@code -c:v libsvtav1 -preset 8 -b:v 150k -g 15} and + * {@code -b:v 3M -g 8}. + */ + private static final String AV1 = "/media-test-av1.mkv"; + + private static final String AV1_1080P = "/media-test-av1-1080p.mkv"; + /** VP8, for which no platform has a decoder this module uses. */ private static final String VP8 = "/media-test.webm"; @@ -120,6 +158,9 @@ class HardwareDecodingTest { /** Whether the hardware decoder of this machine decodes, once it is known. */ private Boolean hardwareDecodes; + /** Whether this machine decodes HEVC in hardware, once it is known. */ + private Boolean hevcHardwareDecodes; + @BeforeAll void initFactory() { @@ -143,6 +184,118 @@ void vp9MatchesSoftware() throws Exception { assertMatchesSoftware(VP9); } + @Test + void hevcMatchesReference() throws Exception { + assertHevcMatchesReference(HEVC, FRAMES, null); + } + + @Test + void hevcMatchesReference1080p() throws Exception { + // HEVC decodes in blocks of up to 64, so the surface is 1088 rows high + // and the player has to crop it. + assertHevcMatchesReference(HEVC_1080P, 15, "1920x1080"); + } + + @Test + void hevcGoesToHardwareUnasked() throws Exception { + requireHevcHardware(); + + Result result = play(HEVC, false, 0); + + assertNull(result.error); + assertTrue(result.hardware); + assertEquals(reference(HEVC), result.frames); + assertEquals(0, HardwareFault.softwareThreads(), "a software decoder was opened"); + } + + @Test + void hevcWithoutHardwareDoesNotOpen() throws Exception { + // On any machine: without a hardware device, HEVC has no decoder, + // asked for hardware or not. + HardwareFault.hideDevices(); + + for (boolean hardware : new boolean[] { true, false }) { + CustomVideoSource source = new CustomVideoSource(); + + try { + IOException error = assertThrows(IOException.class, + () -> new MediaPlayer(new MediaReader(path(HEVC)), source, null, hardware)); + + assertTrue(error.getMessage().contains("Decoder not found"), error.getMessage()); + } + finally { + source.dispose(); + } + } + + assertEquals(0, HardwareFault.softwareThreads(), "a software decoder was opened"); + } + + @Test + void otherCodecsWithoutHardwareFallBack() throws Exception { + // The same hidden devices leave H.264 to software, as ever. + HardwareFault.hideDevices(); + + Result result = play(H264, true, 0); + + assertNull(result.error); + assertFalse(result.hardware); + assertEquals(FRAMES, result.frames.size()); + assertTrue(HardwareFault.softwareThreads() > 0, "no software decoder"); + } + + @Test + void hevcHardwareFailureStopsPlayback() throws Exception { + requireHevcHardware(); + + // Before the first picture, where any other codec would have its + // packets decoded again in software. + HardwareFault.failSend(false, 0); + + Result result = play(HEVC, true, 0); + + assertNotNull(result.error); + assertTrue(result.frames.isEmpty(), "frames: " + result.frames.size()); + assertEquals(0, HardwareFault.softwareThreads(), "a software decoder was opened"); + } + + @Test + void hevcHardwareFailureAfterPicturesStopsPlayback() throws Exception { + requireHevcHardware(); + + HardwareFault.failSend(false, 5); + + Result result = play(HEVC, true, 0); + List reference = reference(HEVC); + + // What the hardware delivered, and nothing after it. + assertNotNull(result.error); + assertTrue(result.frames.size() < reference.size(), "frames: " + result.frames.size()); + assertEquals(reference.subList(0, result.frames.size()), result.frames); + assertEquals(0, HardwareFault.softwareThreads(), "a software decoder was opened"); + } + + @Test + void av1MatchesSoftware() throws Exception { + // The software side is dav1d, which FFmpeg prefers to its own decoder, + // the one that decodes on the hardware. + assertMatchesSoftware(AV1, FRAMES, null, HARDWARE_AV1_REQUIRED); + } + + @Test + void av1MatchesSoftware1080p() throws Exception { + assertMatchesSoftware(AV1_1080P, 15, "1920x1080", HARDWARE_AV1_REQUIRED); + } + + @Test + void av1DecodesInSoftware() throws Exception { + Result result = play(AV1, false, 0); + + assertNull(result.error); + assertFalse(result.hardware); + assertEquals(FRAMES, result.frames.size()); + } + @Test void softwareIsTheDefault() throws Exception { Result result = play(H264, false, 0); @@ -620,13 +773,18 @@ private void assertMatchesSoftware(String asset) throws Exception { } private void assertMatchesSoftware(String asset, int frames, String size) throws Exception { + assertMatchesSoftware(asset, frames, size, HARDWARE_REQUIRED); + } + + private void assertMatchesSoftware(String asset, int frames, String size, boolean required) + throws Exception { Result software = play(asset, false, 0); Result hardware = play(asset, true, 0); assertNull(software.error); assertNull(hardware.error); - requireHardware(hardware); + requireHardware(hardware.hardware, required); assertEquals(frames, software.frames.size()); @@ -639,6 +797,71 @@ private void assertMatchesSoftware(String asset, int frames, String size) throws assertEquals(software.frames, hardware.frames); } + /** + * HEVC, which is decoded in hardware only, has no software decoding to + * compare with: its pictures are compared with checksums of the ones + * FFmpeg's software decoder gave. HEVC decoding is exact, so every + * decoder that follows the standard gives the same. + */ + private void assertHevcMatchesReference(String asset, int frames, String size) throws Exception { + requireHevcHardware(); + + Result hardware = play(asset, true, 0); + + assertNull(hardware.error); + assertTrue(hardware.hardware); + assertEquals(frames, hardware.frames.size()); + + if (size != null) { + assertEquals(size, hardware.size); + } + + assertEquals(reference(asset), hardware.frames); + } + + /** + * Skips the test where this machine does not decode HEVC in hardware, and + * fails it where hardware is required. A decoder that opens is not one that + * decodes (see {@link #requireHardwareDecoder()}), so a stream is played + * first, once. + */ + private void requireHevcHardware() throws Exception { + if (hevcHardwareDecodes == null) { + try { + Result result = play(HEVC, true, 0); + + hevcHardwareDecodes = result.error == null && result.hardware && !result.frames.isEmpty(); + } + catch (IOException e) { + // No hardware decoder for HEVC, so no decoder at all. + hevcHardwareDecodes = false; + } + finally { + HardwareFault.disarm(); + } + } + + requireHardware(hevcHardwareDecodes); + } + + /** + * The luma checksums of an asset's pictures, from the file next to it + * named after it with {@code .luma.txt} in place of its extension, one + * hexadecimal CRC-32 a line. + */ + private static List reference(String asset) throws Exception { + String name = asset.substring(0, asset.lastIndexOf('.')) + ".luma.txt"; + List checksums = new ArrayList<>(); + + for (String line : Files.readAllLines(path(name), StandardCharsets.US_ASCII)) { + if (!line.trim().isEmpty()) { + checksums.add(Long.parseLong(line.trim(), 16)); + } + } + + return checksums; + } + /** * Fails where hardware is required and the player did not use it, and * skips the test where it is not required. @@ -676,7 +899,11 @@ private void requireHardwareContext() throws Exception { } private static void requireHardware(boolean hardware) { - if (HARDWARE_REQUIRED) { + requireHardware(hardware, HARDWARE_REQUIRED); + } + + private static void requireHardware(boolean hardware, boolean required) { + if (required) { assertTrue(hardware, "the player did not decode in hardware"); } else { diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareFault.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareFault.java index b1b086a1d..86019bf32 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareFault.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/HardwareFault.java @@ -54,7 +54,14 @@ private HardwareFault() { static native void failDrain(); /** - * Forgets any fault, and the thread count of {@link #softwareThreads()}. + * Makes the platform look as if it had no hardware device to decode + * with, until {@link #disarm()}: a machine without a GPU decoder. + */ + static native void hideDevices(); + + /** + * Forgets any fault and hidden devices, and the thread count of + * {@link #softwareThreads()}. */ static native void disarm(); diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java index 36fab8926..1cbd4c413 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/MediaRecorderTest.java @@ -151,6 +151,16 @@ void recordsSentCallIntoWebm() throws Exception { assertDuration(info); } + @Test + void recordsMuLawIntoMatroska() throws Exception { + assertRecordsG711("PCMU", "pcm_mulaw"); + } + + @Test + void recordsALawIntoMatroska() throws Exception { + assertRecordsG711("PCMA", "pcm_alaw"); + } + @Test void leavesOutCodecFileCannotHold() throws Exception { // MP4 has no place for VP8, which the call sends. @@ -250,6 +260,40 @@ public void onStarted() { assertTrue(readInfo(file).hasVideo()); } + /** + * Records a call that sends G.711 into Matroska, which has no codec ID of + * its own for it, and reads the audio back as the codec it was sent in. + */ + private void assertRecordsG711(String sdpName, String codecName) throws Exception { + Path file = tempDir.resolve(sdpName + ".mkv"); + Listener listener = new Listener(); + + try (TestCall call = new TestCall(factory, sdpName); + MediaRecorder recorder = new MediaRecorder(file)) { + recorder.setListener(listener); + recorder.addTrack(call.getVideoSender()); + recorder.addTrack(call.getAudioSender()); + recorder.start(); + + assertTrue(listener.started.await(10, TimeUnit.SECONDS), "not started"); + + Thread.sleep(RECORD_MS); + + assertTrue(recorder.stop()); + } + + assertNull(listener.error.get()); + assertTrue(listener.warnings.isEmpty(), listener.warnings.toString()); + + MediaInfo info = readInfo(file); + + assertEquals("vp8", info.getVideoCodec()); + assertEquals(codecName, info.getAudioCodec()); + assertEquals(8000, info.getSampleRate()); + assertEquals(1, info.getChannels()); + assertDuration(info); + } + private static MediaInfo readInfo(Path file) throws Exception { try (MediaReader reader = new MediaReader(file)) { return reader.getInfo(); diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java index e282e7976..e55116d21 100644 --- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/TestCall.java @@ -74,6 +74,14 @@ class TestCall implements AutoCloseable { TestCall(PeerConnectionFactory factory) throws Exception { + this(factory, "opus"); + } + + /** + * A call that sends its audio in the named codec, "opus", "PCMU" or + * "PCMA". + */ + TestCall(PeerConnectionFactory factory, String audioCodec) throws Exception { caller = new Peer(factory); callee = new Peer(factory); @@ -86,7 +94,8 @@ class TestCall implements AutoCloseable { videoSender = caller.connection.addTrack(videoTrack, Collections.singletonList("stream")); audioSender = caller.connection.addTrack(audioTrack, Collections.singletonList("stream")); - preferVp8(factory, caller.connection, videoSender); + prefer(factory, caller.connection, videoSender, MediaType.VIDEO, "VP8"); + prefer(factory, caller.connection, audioSender, MediaType.AUDIO, audioCodec); RTCSessionDescription offer = caller.createOffer(); callee.setRemoteDescription(offer); @@ -145,17 +154,18 @@ public void close() throws InterruptedException { } /** - * Makes VP8 the preferred video codec. Platforms prefer different codecs by - * default, macOS H.264 through VideoToolbox, and the tests should not - * depend on which: VP8 is available everywhere, and it is what the - * recordings are checked for. + * Makes the named codec the preferred one of the sender. Platforms prefer + * different video codecs by default, macOS H.264 through VideoToolbox, and + * the tests should not depend on which: VP8 is available everywhere, and + * it is what the recordings are checked for. Audio is Opus unless a test + * asks for G.711. */ - private static void preferVp8(PeerConnectionFactory factory, RTCPeerConnection connection, - RTCRtpSender sender) { + private static void prefer(PeerConnectionFactory factory, RTCPeerConnection connection, + RTCRtpSender sender, MediaType mediaType, String codecName) { List codecs = new ArrayList<>( - factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); + factory.getRtpSenderCapabilities(mediaType).getCodecs()); - codecs.sort(Comparator.comparing(codec -> !"VP8".equalsIgnoreCase(codec.getName()))); + codecs.sort(Comparator.comparing(codec -> !codecName.equalsIgnoreCase(codec.getName()))); for (RTCRtpTransceiver transceiver : connection.getTransceivers()) { RTCRtpSender transceiverSender = transceiver.getSender(); diff --git a/webrtc-java-media/src/test/resources/media-test-av1-1080p.mkv b/webrtc-java-media/src/test/resources/media-test-av1-1080p.mkv new file mode 100644 index 000000000..feeaaabee Binary files /dev/null and b/webrtc-java-media/src/test/resources/media-test-av1-1080p.mkv differ diff --git a/webrtc-java-media/src/test/resources/media-test-av1.mkv b/webrtc-java-media/src/test/resources/media-test-av1.mkv new file mode 100644 index 000000000..da3ba2867 Binary files /dev/null and b/webrtc-java-media/src/test/resources/media-test-av1.mkv differ diff --git a/webrtc-java-media/src/test/resources/media-test-hevc-1080p.luma.txt b/webrtc-java-media/src/test/resources/media-test-hevc-1080p.luma.txt new file mode 100644 index 000000000..dcfc4349b --- /dev/null +++ b/webrtc-java-media/src/test/resources/media-test-hevc-1080p.luma.txt @@ -0,0 +1,15 @@ +8c3c9e7b +a0d1df01 +f899a913 +e1b2a233 +26731463 +dab74eb6 +8f57e290 +c0488eb +42f00b0a +860010c +fc95a27a +86961a87 +e42145e6 +fdf378bd +3b04dbf6 diff --git a/webrtc-java-media/src/test/resources/media-test-hevc-1080p.mkv b/webrtc-java-media/src/test/resources/media-test-hevc-1080p.mkv new file mode 100644 index 000000000..ed269dbc5 Binary files /dev/null and b/webrtc-java-media/src/test/resources/media-test-hevc-1080p.mkv differ diff --git a/webrtc-java-media/src/test/resources/media-test-hevc.luma.txt b/webrtc-java-media/src/test/resources/media-test-hevc.luma.txt new file mode 100644 index 000000000..d224cf8e4 --- /dev/null +++ b/webrtc-java-media/src/test/resources/media-test-hevc.luma.txt @@ -0,0 +1,30 @@ +12994f9c +67c679cc +6776741a +d7b6ea7f +ae0fb426 +60a50beb +21bc9f85 +17d53f9b +fda65975 +49033653 +f2c6859b +80130c3c +801f19f4 +ae8d5d0c +44b25a5d +e4a7f889 +b564da0e +a43f9e58 +f36d0333 +d85774ae +a0a5e3f1 +e34aa78e +356eb788 +95cbc4 +280adf56 +91a2de21 +202daff7 +17f9ee60 +4e5a259a +63091359 diff --git a/webrtc-java-media/src/test/resources/media-test-hevc.mkv b/webrtc-java-media/src/test/resources/media-test-hevc.mkv new file mode 100644 index 000000000..bcb5d528a Binary files /dev/null and b/webrtc-java-media/src/test/resources/media-test-hevc.mkv differ diff --git a/webrtc-java-media/third-party/dav1d b/webrtc-java-media/third-party/dav1d new file mode 160000 index 000000000..54706fc6b --- /dev/null +++ b/webrtc-java-media/third-party/dav1d @@ -0,0 +1 @@ +Subproject commit 54706fc6bc0cdecab7e9593974a4039cc038fca7