From 358ab2e6dba7c44101f07fce1260c82c1029a60c Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Sun, 11 Oct 2026 09:54:50 +0200 Subject: [PATCH] feat: build WebRTC with H.265 WebRTC was built without H.265 (rtc_use_h265 follows proprietary_codecs, which is off), so a call could not carry it even with an encoder and a decoder for it: no H.265 packetization, no depacketizer (WebRTC creates none and drops the stream) and no H.265 negotiation. It is now built with rtc_use_h265=true, and the JNI code with RTC_ENABLE_H265 to match. WebRTC has no H.265 codec of its own and none is added, so H.265 is offered only by a factory that brings one. A compiled WebRTC is used only if it was built with the current GN args, which are recorded next to the library in the build tree and the install directory. A build without the record or with other args is built again, and the stale installed library is removed so nothing links it meanwhile. Before, any existing library was taken as it was, so a change to the args never reached a machine that had built WebRTC once. The record is written only when gn and ninja succeed. Java encoders get the QP of H.265 frames parsed, and the H.264 quality scaling thresholds for H.265, as WebRTC's Android wrapper has them. A new test sends a call through a Java H.265 encoder and decoder, whose frames are wrapped in H.265 NAL units. --- docs/guide/advanced/video-codecs.md | 18 +- .../cpp/dependencies/webrtc/CMakeLists.txt | 135 ++++++++++---- .../media/video/codec/VideoEncoderWrapper.h | 2 + .../media/video/codec/VideoEncoderWrapper.cpp | 9 + .../media/video/codec/EncodedImage.java | 3 +- .../media/video/codec/VideoEncoder.java | 2 +- .../onvoid/webrtc/VideoCodecFactoryTests.java | 172 ++++++++++++++++-- 7 files changed, 288 insertions(+), 53 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index be5070d76..4cf1d994d 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -112,6 +112,22 @@ class MyEncoderFactory implements VideoEncoderFactory { The list is in order of preference. A codec name WebRTC does not know, such as `X-MYCODEC`, is negotiated like any other and sent with the generic RTP packetization. `RTCRtpTransceiver.setCodecPreferences()` picks among the negotiated codecs per transceiver. +### H.265 + +The library is built with WebRTC's H.265 support: H.265 is negotiated with its SDP parameters (`profile-id`, `tier-flag`, `level-id`, `tx-mode`) and sent with the H.265 RTP payload format (RFC 7798). WebRTC has no H.265 encoder or decoder of its own, and this library adds none in software, so H.265 is offered only by a factory that brings one, such as a hardware encoder reached through another library: + +```java +Map parameters = new HashMap<>(); +parameters.put("profile-id", "1"); // Main +parameters.put("tier-flag", "0"); +parameters.put("level-id", "93"); // level 3.1 +parameters.put("tx-mode", "SRST"); + +codecs.add(new VideoCodecInfo("H265", parameters)); +``` + +The encoder hands WebRTC each picture as an access unit in Annex B form, with the parameter sets (VPS, SPS and PPS) in every key frame, and the decoder receives them the same way. + ## Implementing an Encoder WebRTC calls `initEncode()`, then `encode()` for each frame and `setRates()` whenever the target bitrate changes, and finally `release()`. It may initialize a released encoder again, possibly with another frame size. These calls come from one WebRTC thread at a time. @@ -165,7 +181,7 @@ Things to keep in mind: - The capture time of an `EncodedImage` must be the `timestampNs` of the frame it encodes. That is how WebRTC matches output to input; an image with any other capture time is dropped. - The frame passed to `encode()` is valid only until the method returns. An encoder that works asynchronously calls `frame.retain()` and later `frame.release()`, and hands its output to the callback from its own thread, in encoding order. The pixels must not be changed, since other consumers may share them. - `onEncodedFrame()` copies the payload, so the buffer may be reused once it returns. Frames handed over after `release()` are ignored. -- `getScalingSettings()`, `getEncoderInfo()`, `getResolutionBitrateLimits()`, `getImplementationName()` and `isHardwareEncoder()` have defaults and may be overridden. The default scaling settings use WebRTC's quantizer thresholds for VP8, VP9 and H.264; a custom codec without a quantizer should return `ScalingSettings.OFF`. +- `getScalingSettings()`, `getEncoderInfo()`, `getResolutionBitrateLimits()`, `getImplementationName()` and `isHardwareEncoder()` have defaults and may be overridden. The default scaling settings use WebRTC's quantizer thresholds for VP8, VP9, H.264 and H.265 (which has those of H.264 until WebRTC has its own); a custom codec without a quantizer should return `ScalingSettings.OFF`. ## Implementing a Decoder diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt b/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt index d85aff56a..df204b909 100644 --- a/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt @@ -76,17 +76,22 @@ function(print text) endfunction() function(execute_command) - cmake_parse_arguments(CMD_ARGS "" "WORKING_DIRECTORY" "COMMAND" ${ARGN}) + cmake_parse_arguments(CMD_ARGS "" "WORKING_DIRECTORY;RESULT_VARIABLE" "COMMAND" ${ARGN}) execute_process( COMMAND ${SHELLCOMMAND} ${CMD_ARGS_COMMAND} WORKING_DIRECTORY "${CMD_ARGS_WORKING_DIRECTORY}" OUTPUT_VARIABLE COMMAND_OUTPUT ERROR_VARIABLE COMMAND_ERROR + RESULT_VARIABLE COMMAND_RESULT ) print("${COMMAND_OUTPUT}") print("${COMMAND_ERROR}") + + if(CMD_ARGS_RESULT_VARIABLE) + set(${CMD_ARGS_RESULT_VARIABLE} "${COMMAND_RESULT}" PARENT_SCOPE) + endif() endfunction() function(sysroot_install) @@ -130,6 +135,74 @@ message(STATUS "WebRTC build type: ${CMAKE_BUILD_TYPE}") message(STATUS "WebRTC use custom libcxx: ${CUSTOM_LIBCXX}") message(STATUS "WebRTC install path: ${WEBRTC_INSTALL_DIR}") +# The GN args WebRTC is built with. They come before the check for a compiled +# WebRTC below, because a build is only used if it was made with these. +set(COMPILE_ARGS "target_cpu=\"${TARGET_CPU}\" \ +use_custom_libcxx=${CUSTOM_LIBCXX} \ +is_clang=true \ +is_debug=false \ +is_component_build=false \ +treat_warnings_as_errors=false \ +rtc_build_tools=false \ +rtc_use_perfetto=false \ +rtc_use_pipewire=false \ +rtc_enable_protobuf=false \ +rtc_build_examples=false \ +rtc_include_tests=false \ +use_rtti=true \ +rtc_use_h264=true \ +rtc_use_h265=true \ +ffmpeg_branding=\"Chrome\" \ +symbol_level=0") + +if(APPLE) + set(COMPILE_ARGS "${COMPILE_ARGS} mac_deployment_target=\"${CMAKE_OSX_DEPLOYMENT_TARGET}\"") +elseif(LINUX) + # Build the PipeWire/xdg-desktop-portal capturer for Wayland sessions. The + # X11 capturer stays in as well; WebRTC picks PipeWire at runtime only when + # IsRunningUnderWayland() is true. rtc_link_pipewire stays false (default), + # so libpipewire is dlopen'd and X11-only systems gain no hard dependency. + string(REPLACE "rtc_use_pipewire=false" "rtc_use_pipewire=true" COMPILE_ARGS "${COMPILE_ARGS}") + + if(${TARGET_CPU} STREQUAL "arm") + set(COMPILE_ARGS "${COMPILE_ARGS} custom_toolchain=\"//build/toolchain/linux:clang_arm\"") + elseif(${TARGET_CPU} STREQUAL "arm64") + set(COMPILE_ARGS "${COMPILE_ARGS} custom_toolchain=\"//build/toolchain/linux:clang_arm64\"") + endif() +endif() + +# Written next to a compiled WebRTC, in the build tree and in the install +# directory, naming the GN args it was built with. A build without it, or with +# other args, is not used and WebRTC is built again: it would lack what the +# args enable (H.265 for one), or have defines, and with them struct layouts, +# that differ from those the JNI code is compiled with below. Changing an arg +# here thus rebuilds WebRTC on its own; CI's cache key covers this file. +set(WEBRTC_ARGS_FILE "webrtc-java-gn-args.txt") + +function(webrtc_args_match DIRECTORY RESULT) + set(MATCHES FALSE) + + if(EXISTS "${DIRECTORY}/${WEBRTC_ARGS_FILE}") + file(READ "${DIRECTORY}/${WEBRTC_ARGS_FILE}" RECORDED_ARGS) + + if(RECORDED_ARGS STREQUAL COMPILE_ARGS) + set(MATCHES TRUE) + endif() + endif() + + set(${RESULT} ${MATCHES} PARENT_SCOPE) +endfunction() + +webrtc_args_match("${WEBRTC_INSTALL_DIR}" WEBRTC_INSTALLED_ARGS_MATCH) +webrtc_args_match("${WEBRTC_SRC}/${WEBRTC_BUILD}" WEBRTC_BUILT_ARGS_MATCH) + +if(EXISTS "${WEBRTC_LIB_PATH_INSTALLED}" AND NOT WEBRTC_INSTALLED_ARGS_MATCH) + # Out of the way, so that nothing links it before the new build replaces + # it on install. + message(STATUS "WebRTC: the installed build has other GN args, building again") + file(REMOVE "${WEBRTC_LIB_PATH_INSTALLED}") +endif() + # The api/video utility classes (AdaptedVideoTrackSource, VideoAdapter, # VideoBroadcaster) fell out of the default build graph when they moved out @@ -197,6 +270,11 @@ target_include_directories(${PROJECT_NAME} ) target_link_libraries(${PROJECT_NAME} ${TARGET_LINK_LIB} ${VIDEO_UTIL_LINK_LIBS}) +# rtc_use_h265=true makes gn define RTC_ENABLE_H265 for WebRTC; the JNI code +# needs it as well, for the H.265 parts of WebRTC's headers it uses (the H.265 +# bitstream parser) and to be compiled against the same declarations. +target_compile_definitions(${PROJECT_NAME} PUBLIC RTC_ENABLE_H265) + if(APPLE) target_include_directories(${PROJECT_NAME} PUBLIC @@ -240,7 +318,8 @@ elseif(WIN32) target_link_libraries(${PROJECT_NAME} D3D11 DXGI user32 gdi32 iphlpapi dmoguids msdmo secur32 strmiids winmm wmcodecdspuuid ws2_32) endif() -if(EXISTS "${WEBRTC_LIB_PATH}" OR EXISTS "${WEBRTC_LIB_PATH_INSTALLED}") +if((EXISTS "${WEBRTC_LIB_PATH}" AND WEBRTC_BUILT_ARGS_MATCH) OR + (EXISTS "${WEBRTC_LIB_PATH_INSTALLED}" AND WEBRTC_INSTALLED_ARGS_MATCH)) message(STATUS "WebRTC: Compiled version found '${TARGET_LINK_LIB}'") if(LINUX) @@ -350,38 +429,6 @@ if (PATCHES) endif() message(STATUS "WebRTC: generate") -set(COMPILE_ARGS "target_cpu=\"${TARGET_CPU}\" \ -use_custom_libcxx=${CUSTOM_LIBCXX} \ -is_clang=true \ -is_debug=false \ -is_component_build=false \ -treat_warnings_as_errors=false \ -rtc_build_tools=false \ -rtc_use_perfetto=false \ -rtc_use_pipewire=false \ -rtc_enable_protobuf=false \ -rtc_build_examples=false \ -rtc_include_tests=false \ -use_rtti=true \ -rtc_use_h264=true \ -ffmpeg_branding=\"Chrome\" \ -symbol_level=0") - -if(APPLE) - set(COMPILE_ARGS "${COMPILE_ARGS} mac_deployment_target=\"${CMAKE_OSX_DEPLOYMENT_TARGET}\"") -elseif(LINUX) - # Build the PipeWire/xdg-desktop-portal capturer for Wayland sessions. The - # X11 capturer stays in as well; WebRTC picks PipeWire at runtime only when - # IsRunningUnderWayland() is true. rtc_link_pipewire stays false (default), - # so libpipewire is dlopen'd and X11-only systems gain no hard dependency. - string(REPLACE "rtc_use_pipewire=false" "rtc_use_pipewire=true" COMPILE_ARGS "${COMPILE_ARGS}") - - if(${TARGET_CPU} STREQUAL "arm") - set(COMPILE_ARGS "${COMPILE_ARGS} custom_toolchain=\"//build/toolchain/linux:clang_arm\"") - elseif(${TARGET_CPU} STREQUAL "arm64") - set(COMPILE_ARGS "${COMPILE_ARGS} custom_toolchain=\"//build/toolchain/linux:clang_arm64\"") - endif() -endif() if(LINUX) # The sysroot pkg-config environment exported for the JNI configure step @@ -395,9 +442,14 @@ if(LINUX) unset(ENV{PKG_CONFIG_PATH}) endif() +# The record of the GN args goes away first: a build that fails part way +# leaves a library that is neither the old one nor the new. +file(REMOVE "${WEBRTC_SRC}/${WEBRTC_BUILD}/${WEBRTC_ARGS_FILE}") + execute_command( COMMAND gn gen ${WEBRTC_BUILD} --args=${COMPILE_ARGS} WORKING_DIRECTORY "${WEBRTC_SRC}" + RESULT_VARIABLE GN_RESULT ) message(STATUS "WebRTC: compile") @@ -405,7 +457,9 @@ if(APPLE) execute_command( COMMAND ninja -C "${WEBRTC_BUILD}" :default api/audio_codecs:builtin_audio_decoder_factory api/task_queue:default_task_queue_factory sdk:native_api sdk:default_codec_factory_objc pc:peer_connection sdk:videocapture_objc api/video:adapted_video_track_source api/video:video_adapter api/video:video_broadcaster WORKING_DIRECTORY "${WEBRTC_SRC}" + RESULT_VARIABLE NINJA_RESULT ) + set(NINJA_UTIL_RESULT 0) file(GLOB_RECURSE SDK_OBJC_FILES ${WEBRTC_SRC}/${WEBRTC_BUILD}/obj/*.o) @@ -416,6 +470,7 @@ else() execute_command( COMMAND ninja -C "${WEBRTC_BUILD}" WORKING_DIRECTORY "${WEBRTC_SRC}" + RESULT_VARIABLE NINJA_RESULT ) # The api/video utility targets are not reachable from the default @@ -424,6 +479,7 @@ else() execute_command( COMMAND ninja -C "${WEBRTC_BUILD}" api/video:adapted_video_track_source api/video:video_adapter api/video:video_broadcaster WORKING_DIRECTORY "${WEBRTC_SRC}" + RESULT_VARIABLE NINJA_UTIL_RESULT ) endif() @@ -518,6 +574,17 @@ elseif(WIN32) endforeach() endif() +# The record of the GN args, once the build has succeeded, in the build tree +# and, with the library, in the install directory. +if(GN_RESULT EQUAL 0 AND NINJA_RESULT EQUAL 0 AND NINJA_UTIL_RESULT EQUAL 0 AND EXISTS "${WEBRTC_LIB_PATH}") + file(WRITE "${WEBRTC_SRC}/${WEBRTC_BUILD}/${WEBRTC_ARGS_FILE}" "${COMPILE_ARGS}") + + install(FILES "${WEBRTC_SRC}/${WEBRTC_BUILD}/${WEBRTC_ARGS_FILE}" DESTINATION "${WEBRTC_INSTALL_DIR}") +else() + message(WARNING "WebRTC: the build failed (gn ${GN_RESULT}, ninja ${NINJA_RESULT}, ${NINJA_UTIL_RESULT}); " + "its GN args are not recorded, so the next configure builds it again") +endif() + install(FILES "${WEBRTC_LIB_PATH}" DESTINATION "${WEBRTC_INSTALL_DIR}/lib") install( DIRECTORY "${WEBRTC_SRC}/" diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h index 57df254c3..89c906dce 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h @@ -26,6 +26,7 @@ #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "common_video/h264/h264_bitstream_parser.h" +#include "common_video/h265/h265_bitstream_parser.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/svc/scalable_video_controller_no_layering.h" @@ -155,6 +156,7 @@ namespace jni // Used on the thread the Java encoder hands its frames over on. webrtc::H264BitstreamParser h264BitstreamParser; + webrtc::H265BitstreamParser h265BitstreamParser; webrtc::ScalableVideoControllerNoLayering svcController; webrtc::GofInfoVP9 gof; size_t gofIndex; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp index ff500532f..e1c2dff6e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp @@ -502,6 +502,9 @@ namespace jni case webrtc::kVideoCodecVP9: return ScalingSettings(low.value_or(kLowVp9QpThreshold), high.value_or(kHighVp9QpThreshold)); + case webrtc::kVideoCodecH265: + // WebRTC has no thresholds of its own for H.265 yet and uses those + // of H.264 (bugs.webrtc.org/13485), as its Android wrapper does. case webrtc::kVideoCodecH264: return ScalingSettings(low.value_or(kLowH264QpThreshold), high.value_or(kHighH264QpThreshold)); @@ -564,6 +567,12 @@ namespace jni success = qp >= 0; break; + case webrtc::kVideoCodecH265: + h265BitstreamParser.ParseBitstream(image); + qp = h265BitstreamParser.GetLastSliceQp().value_or(-1); + success = qp >= 0; + break; + default: break; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java index 6a9f9a97d..855d4f965 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java @@ -312,7 +312,8 @@ public Builder setRotation(int rotation) { /** * Sets the quantizer. Without one, WebRTC parses it from the payload - * of VP8, VP9 and H.264 frames, which quality scaling depends on. + * of VP8, VP9, H.264 and H.265 frames, which quality scaling depends + * on. * * @param qp The quantizer, or {@code null} if it is not known. * diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java index c76bf9793..445a603d8 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java @@ -278,7 +278,7 @@ class ScalingSettings { /** * Quality scaling with the thresholds WebRTC uses for VP8, VP9 and - * H.264, and none for any other codec. + * H.264, and for H.265 those of H.264, and none for any other codec. */ public static final ScalingSettings DEFAULT = new ScalingSettings(true, null, null); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java index c45141b6e..f4abff11c 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -43,9 +43,13 @@ import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -70,6 +74,15 @@ class VideoCodecFactoryTests extends TestBase { /** The codec of {@link TestEncoder} and {@link TestDecoder}. */ private static final String TEST_CODEC = "X-TEST"; + /** + * H.265 as WebRTC offers it, which {@link TestEncoder} and {@link + * TestDecoder} speak as well, framing their payload in H.265 NAL units. + */ + private static final VideoCodecInfo H265 = new VideoCodecInfo("H265", h265Parameters()); + + /** What precedes the payload in the slice of an H.265 test access unit. */ + private static final String H265_MARKER = "WRTC:"; + @Test void defaultFactoriesListBuiltInCodecs() { @@ -132,23 +145,31 @@ void customCodecIsAdvertised() { @Test void javaCodecRoundTrips() throws Exception { - assertRoundTrip(false); + assertRoundTrip(new VideoCodecInfo(TEST_CODEC), false); } @Test void javaFrameBufferRoundTrips() throws Exception { // The decoder hands back frames whose buffer is no NativeI420Buffer, // which WebRTC has to copy. - assertRoundTrip(true); + assertRoundTrip(new VideoCodecInfo(TEST_CODEC), true); + } + + @Test + void h265CallRoundTrips() throws Exception { + // WebRTC has no H.265 codec of its own, so this is H.265 as a codec + // factory brings it: negotiated, packetized into RTP and depacketized + // as H.265, which WebRTC does only when built with it. + assertRoundTrip(H265, false); } /** * Sends a call through the test codec and checks that every frame * received is one that was sent, and how WebRTC used the codecs. */ - private void assertRoundTrip(boolean javaBuffers) throws Exception { - TestEncoderFactory encoderFactory = new TestEncoderFactory(); - TestDecoderFactory decoderFactory = new TestDecoderFactory(); + private void assertRoundTrip(VideoCodecInfo codec, boolean javaBuffers) throws Exception { + TestEncoderFactory encoderFactory = new TestEncoderFactory(codec); + TestDecoderFactory decoderFactory = new TestDecoderFactory(codec); decoderFactory.javaBuffers = javaBuffers; PeerConnectionFactory built = testCodecFactory(encoderFactory, decoderFactory); @@ -162,7 +183,7 @@ private void assertRoundTrip(boolean javaBuffers) throws Exception { encodedLuma.add(luma); }; - try (TestMediaCall call = new TestMediaCall(built, true, false, TEST_CODEC)) { + try (TestMediaCall call = new TestMediaCall(built, true, false, codec.getName())) { call.negotiate(); RTCRtpReceiver receiver = call.getReceiver("video"); @@ -369,6 +390,97 @@ private static List names(List codecs) { .collect(Collectors.toList()); } + /** The SDP parameters of H.265 Main profile, main tier, level 3.1. */ + private static Map h265Parameters() { + Map parameters = new HashMap<>(); + parameters.put("profile-id", "1"); + parameters.put("tier-flag", "0"); + parameters.put("level-id", "93"); + parameters.put("tx-mode", "SRST"); + return parameters; + } + + private static boolean isH265(VideoCodecInfo codec) { + return H265.getName().equalsIgnoreCase(codec.getName()); + } + + /** + * Frames a payload as an H.265 access unit, in Annex B: the parameter sets + * WebRTC wants with every key frame (VPS, SPS, PPS), then an IDR slice that + * starts a picture and carries the payload. The payload goes in as hex + * text after a marker, so that no byte of it is zero and nothing in it + * reads as a start code. WebRTC looks at the NAL unit types only; the + * parameter sets need not make sense. + */ + static ByteBuffer toH265AccessUnit(ByteBuffer payload) { + StringBuilder text = new StringBuilder(H265_MARKER); + + while (payload.hasRemaining()) { + text.append(String.format("%02X", payload.get() & 0xFF)); + } + + byte[] slice = text.toString().getBytes(StandardCharsets.US_ASCII); + byte[][] nalUnits = { + // The NAL unit header: the type in bits 1 to 6 of the first byte, + // then layer 0 and temporal ID 1. + { 0x40, 0x01, 0x0C, 0x01, (byte) 0xFF }, // VPS (32) + { 0x42, 0x01, 0x01, 0x01, (byte) 0xFF }, // SPS (33) + { 0x44, 0x01, (byte) 0xC1, 0x72 }, // PPS (34) + // IDR_W_RADL (19); the first bit of the slice header says that it + // is the first slice of its picture. + concat(new byte[] { 0x26, 0x01, (byte) 0x80 }, slice) + }; + + int size = 0; + + for (byte[] nalUnit : nalUnits) { + size += 4 + nalUnit.length; + } + + ByteBuffer accessUnit = ByteBuffer.allocateDirect(size); + + for (byte[] nalUnit : nalUnits) { + accessUnit.put(new byte[] { 0, 0, 0, 1 }).put(nalUnit); + } + + accessUnit.flip(); + + return accessUnit; + } + + /** + * The payload an access unit of {@link #toH265AccessUnit} carries, or + * null if there is none. + */ + static ByteBuffer fromH265AccessUnit(ByteBuffer accessUnit) { + byte[] bytes = new byte[accessUnit.remaining()]; + accessUnit.duplicate().get(bytes); + + String text = new String(bytes, StandardCharsets.ISO_8859_1); + int start = text.indexOf(H265_MARKER); + + if (start < 0) { + return null; + } + + String hex = text.substring(start + H265_MARKER.length()); + ByteBuffer payload = ByteBuffer.allocate(hex.length() / 2); + + for (int i = 0; i + 1 < hex.length(); i += 2) { + payload.put((byte) Integer.parseInt(hex.substring(i, i + 2), 16)); + } + + payload.flip(); + + return payload; + } + + private static byte[] concat(byte[] first, byte[] second) { + byte[] result = Arrays.copyOf(first, first.length + second.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + interface EncodeListener { @@ -387,14 +499,23 @@ static class TestEncoderFactory implements VideoEncoderFactory { final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); final AtomicInteger created = new AtomicInteger(); final AtomicReference last = new AtomicReference<>(); + final VideoCodecInfo codec; volatile EncodeListener onEncode; + TestEncoderFactory() { + this(new VideoCodecInfo(TEST_CODEC)); + } + + TestEncoderFactory(VideoCodecInfo codec) { + this.codec = codec; + } + @Override public List getSupportedCodecs() { List codecs = new ArrayList<>(); - codecs.add(new VideoCodecInfo(TEST_CODEC)); + codecs.add(codec); codecs.addAll(builtIn.getSupportedCodecs()); return codecs; } @@ -403,11 +524,11 @@ public List getSupportedCodecs() { public VideoEncoder createEncoder(VideoCodecInfo info) { created.incrementAndGet(); - if (!TEST_CODEC.equalsIgnoreCase(info.getName())) { + if (!codec.getName().equalsIgnoreCase(info.getName())) { return builtIn.createEncoder(info); } - TestEncoder encoder = new TestEncoder(this); + TestEncoder encoder = new TestEncoder(this, isH265(codec)); last.set(encoder); return encoder; } @@ -424,14 +545,23 @@ static class TestDecoderFactory implements VideoDecoderFactory { final DefaultVideoDecoderFactory builtIn = new DefaultVideoDecoderFactory(); final AtomicInteger created = new AtomicInteger(); final AtomicReference last = new AtomicReference<>(); + final VideoCodecInfo codec; volatile boolean javaBuffers; + TestDecoderFactory() { + this(new VideoCodecInfo(TEST_CODEC, Collections.emptyMap())); + } + + TestDecoderFactory(VideoCodecInfo codec) { + this.codec = codec; + } + @Override public List getSupportedCodecs() { List codecs = new ArrayList<>(); - codecs.add(new VideoCodecInfo(TEST_CODEC, Collections.emptyMap())); + codecs.add(codec); codecs.addAll(builtIn.getSupportedCodecs()); return codecs; } @@ -440,11 +570,11 @@ public List getSupportedCodecs() { public VideoDecoder createDecoder(VideoCodecInfo info) { created.incrementAndGet(); - if (!TEST_CODEC.equalsIgnoreCase(info.getName())) { + if (!codec.getName().equalsIgnoreCase(info.getName())) { return builtIn.createDecoder(info); } - TestDecoder decoder = new TestDecoder(javaBuffers); + TestDecoder decoder = new TestDecoder(javaBuffers, isH265(codec)); last.set(decoder); return decoder; } @@ -461,6 +591,8 @@ static class TestEncoder implements VideoEncoder { private final TestEncoderFactory factory; + private final boolean h265; + final AtomicInteger rates = new AtomicInteger(); final AtomicReference released = new AtomicReference<>(false); @@ -469,8 +601,9 @@ static class TestEncoder implements VideoEncoder { private Callback callback; - TestEncoder(TestEncoderFactory factory) { + TestEncoder(TestEncoderFactory factory, boolean h265) { this.factory = factory; + this.h265 = h265; } @Override @@ -502,6 +635,10 @@ public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { payload.putInt(width).putInt(height).put((byte) luma); payload.flip(); + if (h265) { + payload = toH265AccessUnit(payload); + } + callback.onEncodedFrame(EncodedImage.builder() .setBuffer(payload) .setEncodedWidth(width) @@ -541,11 +678,14 @@ static class TestDecoder implements VideoDecoder { private final boolean javaBuffers; + private final boolean h265; + private Callback callback; - TestDecoder(boolean javaBuffers) { + TestDecoder(boolean javaBuffers, boolean h265) { this.javaBuffers = javaBuffers; + this.h265 = h265; } @Override @@ -562,9 +702,9 @@ public VideoCodecStatus release() { @Override public VideoCodecStatus decode(EncodedImage image) { - ByteBuffer payload = image.getBuffer(); + ByteBuffer payload = h265 ? fromH265AccessUnit(image.getBuffer()) : image.getBuffer(); - if (payload.remaining() != 9) { + if (payload == null || payload.remaining() != 9) { return VideoCodecStatus.ERR_PARAMETER; }