diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h index 89c906dc..7047e187 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h @@ -80,6 +80,8 @@ namespace jni int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method); void InvalidateCallback(JNIEnv * env); void UpdateEncoderInfo(JNIEnv * env); + // Tells WebRTC that a frame given to the encoder will not come out. + void ReportDroppedFrame(uint32_t rtpTimestamp); ScalingSettings GetScalingSettings(JNIEnv * env) const; std::vector GetResolutionBitrateLimits(JNIEnv * env) const; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index c541f6f0..26c7198f 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -164,6 +164,9 @@ namespace jni } image.SetEncodedData(webrtc::EncodedImageBuffer::Create(output.data(), output.size())); + // A stream without layers: nothing else follows for this frame, which + // lets WebRTC release what it keeps of the frame until then. + image.set_end_of_temporal_unit(true); image.set_frame_type(keyFrame ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp index e1c2dff6..b4f8daa9 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp @@ -239,6 +239,8 @@ namespace jni if (jFrame == nullptr) { RTC_LOG(LS_WARNING) << "Java encoder: dropping a frame that cannot be converted to I420"; + ReportDroppedFrame(frame.rtp_timestamp()); + return WEBRTC_VIDEO_CODEC_OK; } @@ -268,10 +270,13 @@ namespace jni result = WEBRTC_VIDEO_CODEC_ERROR; } else { + std::optional givenUp; + { webrtc::MutexLock lock(&frameExtraInfosLock); if (frameExtraInfos.size() >= kMaxPendingFrames) { + givenUp = frameExtraInfos.front().rtpTimestamp; frameExtraInfos.pop_front(); } @@ -281,6 +286,10 @@ namespace jni }); } + if (givenUp) { + ReportDroppedFrame(*givenUp); + } + jobject status = env->CallObjectMethod(encoder, javaClass->encode, jFrame, info); result = ToNativeCodecStatus(env, status, "encode"); } @@ -359,7 +368,8 @@ namespace jni webrtc::EncodedImage image = EncodedImage::toNative(env, imageRef); const int64_t captureTimeNs = EncodedImage::getCaptureTimeNs(env, imageRef); - FrameExtraInfo extraInfo; + std::optional extraInfo; + std::vector dropped; { webrtc::MutexLock lock(&frameExtraInfosLock); @@ -367,21 +377,31 @@ namespace jni // Frames come out in the order they went in, though an encoder // may drop some, so the records of older frames are stale. while (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs < captureTimeNs) { + dropped.push_back(frameExtraInfos.front().rtpTimestamp); frameExtraInfos.pop_front(); } - if (frameExtraInfos.empty() || frameExtraInfos.front().captureTimeNs != captureTimeNs) { - RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time " - << captureTimeNs << " ns"; - return; + if (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs == captureTimeNs) { + extraInfo = frameExtraInfos.front(); + frameExtraInfos.pop_front(); } + } - extraInfo = frameExtraInfos.front(); - frameExtraInfos.pop_front(); + for (uint32_t rtpTimestamp : dropped) { + ReportDroppedFrame(rtpTimestamp); } - image.SetRtpTimestamp(extraInfo.rtpTimestamp); + if (!extraInfo) { + RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time " + << captureTimeNs << " ns"; + return; + } + + image.SetRtpTimestamp(extraInfo->rtpTimestamp); image.capture_time_ms_ = captureTimeNs / webrtc::kNumNanosecsPerMillisec; + // The Java encoders produce a single layer, so every frame ends its + // temporal unit. + image.set_end_of_temporal_unit(true); if (image.qp_ < 0) { image.qp_ = ParseQp(image); @@ -395,6 +415,16 @@ namespace jni } } + void VideoEncoderWrapper::ReportDroppedFrame(uint32_t rtpTimestamp) + { + webrtc::EncodedImageCallback * encodeCallback = callback.load(); + + if (encodeCallback != nullptr) { + // A single layer, so nothing else follows for the frame. + encodeCallback->OnFrameDropped(rtpTimestamp, /*spatial_id=*/0, /*is_end_of_temporal_unit=*/true); + } + } + int32_t VideoEncoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method) { if (status >= 0) { diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java index f4abff11..43614f0c 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -228,6 +228,26 @@ private void assertRoundTrip(VideoCodecInfo codec, boolean javaBuffers) throws E assertTrue(decoderFactory.last.get().released.get(), "decoder was never released"); } + @Test + void javaEncoderDroppingFramesKeepsSending() throws Exception { + // A Java encoder drops a frame by producing nothing for it. WebRTC is + // told about the frames dropped once a later one comes out, and the + // frames encoded still arrive. + TestEncoderFactory encoderFactory = new TestEncoderFactory(); + encoderFactory.dropEveryOtherFrame = true; + + PeerConnectionFactory built = testCodecFactory(encoderFactory, new TestDecoderFactory()); + + try { + assertFramesReceived(built, TEST_CODEC); + } + finally { + built.dispose(); + } + + assertTrue(encoderFactory.dropped.get() > 0, "the encoder dropped no frames"); + } + @Test void factoryDelegatesToBuiltInCodec() throws Exception { AtomicInteger encoders = new AtomicInteger(); @@ -499,10 +519,14 @@ static class TestEncoderFactory implements VideoEncoderFactory { final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); final AtomicInteger created = new AtomicInteger(); final AtomicReference last = new AtomicReference<>(); + final AtomicInteger dropped = new AtomicInteger(); final VideoCodecInfo codec; volatile EncodeListener onEncode; + /** Makes the encoders produce nothing for every other frame. */ + volatile boolean dropEveryOtherFrame; + TestEncoderFactory() { this(new VideoCodecInfo(TEST_CODEC)); @@ -600,6 +624,8 @@ static class TestEncoder implements VideoEncoder { private Callback callback; + private int frames; + TestEncoder(TestEncoderFactory factory, boolean h265) { this.factory = factory; @@ -621,6 +647,12 @@ public VideoCodecStatus release() { @Override public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + if (factory.dropEveryOtherFrame && frames++ % 2 == 1) { + // Dropped: no output, as a rate controlled encoder would do. + factory.dropped.incrementAndGet(); + return VideoCodecStatus.OK; + } + int width = frame.buffer.getWidth(); int height = frame.buffer.getHeight(); int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF;