From fde423a157c8a823ccded6f7b322bbd4f668b42d Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 00:52:44 +0200 Subject: [PATCH] fix: mark encoded frames as the end of their temporal unit Since M152 WebRTC expects encoders to set EncodedImage's end-of-temporal-unit flag, and to report every frame they drop through EncodedImageCallback::OnFrameDropped, so that it can release what it keeps of a frame (the captured frame held for corruption detection, for one) as soon as nothing more of it will come out. All encoders here produce a single layer, so every frame ends its temporal unit. EncoderOutputProcessor sets the flag for the hardware encoders, and the Java encoder wrapper sets it for the frames of Java encoders. The wrapper also reports the frames it gives up on: frames a Java encoder produced no output for, the oldest frame when too many are pending, and frames that cannot be converted for the Java encoder. The Media Foundation and VideoToolbox encoders report their drops already; NVENC and VA-API drop none. --- .../media/video/codec/VideoEncoderWrapper.h | 2 + .../video/codec/EncoderOutputProcessor.cpp | 3 ++ .../media/video/codec/VideoEncoderWrapper.cpp | 46 +++++++++++++++---- .../onvoid/webrtc/VideoCodecFactoryTests.java | 32 +++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h index 89c906dc..7047e187 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h @@ -80,6 +80,8 @@ namespace jni int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method); void InvalidateCallback(JNIEnv * env); void UpdateEncoderInfo(JNIEnv * env); + // Tells WebRTC that a frame given to the encoder will not come out. + void ReportDroppedFrame(uint32_t rtpTimestamp); ScalingSettings GetScalingSettings(JNIEnv * env) const; std::vector GetResolutionBitrateLimits(JNIEnv * env) const; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index c541f6f0..26c7198f 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -164,6 +164,9 @@ namespace jni } image.SetEncodedData(webrtc::EncodedImageBuffer::Create(output.data(), output.size())); + // A stream without layers: nothing else follows for this frame, which + // lets WebRTC release what it keeps of the frame until then. + image.set_end_of_temporal_unit(true); image.set_frame_type(keyFrame ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp index e1c2dff6..b4f8daa9 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp @@ -239,6 +239,8 @@ namespace jni if (jFrame == nullptr) { RTC_LOG(LS_WARNING) << "Java encoder: dropping a frame that cannot be converted to I420"; + ReportDroppedFrame(frame.rtp_timestamp()); + return WEBRTC_VIDEO_CODEC_OK; } @@ -268,10 +270,13 @@ namespace jni result = WEBRTC_VIDEO_CODEC_ERROR; } else { + std::optional givenUp; + { webrtc::MutexLock lock(&frameExtraInfosLock); if (frameExtraInfos.size() >= kMaxPendingFrames) { + givenUp = frameExtraInfos.front().rtpTimestamp; frameExtraInfos.pop_front(); } @@ -281,6 +286,10 @@ namespace jni }); } + if (givenUp) { + ReportDroppedFrame(*givenUp); + } + jobject status = env->CallObjectMethod(encoder, javaClass->encode, jFrame, info); result = ToNativeCodecStatus(env, status, "encode"); } @@ -359,7 +368,8 @@ namespace jni webrtc::EncodedImage image = EncodedImage::toNative(env, imageRef); const int64_t captureTimeNs = EncodedImage::getCaptureTimeNs(env, imageRef); - FrameExtraInfo extraInfo; + std::optional extraInfo; + std::vector dropped; { webrtc::MutexLock lock(&frameExtraInfosLock); @@ -367,21 +377,31 @@ namespace jni // Frames come out in the order they went in, though an encoder // may drop some, so the records of older frames are stale. while (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs < captureTimeNs) { + dropped.push_back(frameExtraInfos.front().rtpTimestamp); frameExtraInfos.pop_front(); } - if (frameExtraInfos.empty() || frameExtraInfos.front().captureTimeNs != captureTimeNs) { - RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time " - << captureTimeNs << " ns"; - return; + if (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs == captureTimeNs) { + extraInfo = frameExtraInfos.front(); + frameExtraInfos.pop_front(); } + } - extraInfo = frameExtraInfos.front(); - frameExtraInfos.pop_front(); + for (uint32_t rtpTimestamp : dropped) { + ReportDroppedFrame(rtpTimestamp); } - image.SetRtpTimestamp(extraInfo.rtpTimestamp); + if (!extraInfo) { + RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time " + << captureTimeNs << " ns"; + return; + } + + image.SetRtpTimestamp(extraInfo->rtpTimestamp); image.capture_time_ms_ = captureTimeNs / webrtc::kNumNanosecsPerMillisec; + // The Java encoders produce a single layer, so every frame ends its + // temporal unit. + image.set_end_of_temporal_unit(true); if (image.qp_ < 0) { image.qp_ = ParseQp(image); @@ -395,6 +415,16 @@ namespace jni } } + void VideoEncoderWrapper::ReportDroppedFrame(uint32_t rtpTimestamp) + { + webrtc::EncodedImageCallback * encodeCallback = callback.load(); + + if (encodeCallback != nullptr) { + // A single layer, so nothing else follows for the frame. + encodeCallback->OnFrameDropped(rtpTimestamp, /*spatial_id=*/0, /*is_end_of_temporal_unit=*/true); + } + } + int32_t VideoEncoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method) { if (status >= 0) { diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java index f4abff11..43614f0c 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -228,6 +228,26 @@ private void assertRoundTrip(VideoCodecInfo codec, boolean javaBuffers) throws E assertTrue(decoderFactory.last.get().released.get(), "decoder was never released"); } + @Test + void javaEncoderDroppingFramesKeepsSending() throws Exception { + // A Java encoder drops a frame by producing nothing for it. WebRTC is + // told about the frames dropped once a later one comes out, and the + // frames encoded still arrive. + TestEncoderFactory encoderFactory = new TestEncoderFactory(); + encoderFactory.dropEveryOtherFrame = true; + + PeerConnectionFactory built = testCodecFactory(encoderFactory, new TestDecoderFactory()); + + try { + assertFramesReceived(built, TEST_CODEC); + } + finally { + built.dispose(); + } + + assertTrue(encoderFactory.dropped.get() > 0, "the encoder dropped no frames"); + } + @Test void factoryDelegatesToBuiltInCodec() throws Exception { AtomicInteger encoders = new AtomicInteger(); @@ -499,10 +519,14 @@ static class TestEncoderFactory implements VideoEncoderFactory { final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); final AtomicInteger created = new AtomicInteger(); final AtomicReference last = new AtomicReference<>(); + final AtomicInteger dropped = new AtomicInteger(); final VideoCodecInfo codec; volatile EncodeListener onEncode; + /** Makes the encoders produce nothing for every other frame. */ + volatile boolean dropEveryOtherFrame; + TestEncoderFactory() { this(new VideoCodecInfo(TEST_CODEC)); @@ -600,6 +624,8 @@ static class TestEncoder implements VideoEncoder { private Callback callback; + private int frames; + TestEncoder(TestEncoderFactory factory, boolean h265) { this.factory = factory; @@ -621,6 +647,12 @@ public VideoCodecStatus release() { @Override public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + if (factory.dropEveryOtherFrame && frames++ % 2 == 1) { + // Dropped: no output, as a rate controlled encoder would do. + factory.dropped.incrementAndGet(); + return VideoCodecStatus.OK; + } + int width = frame.buffer.getWidth(); int height = frame.buffer.getHeight(); int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF;