fix: mark encoded frames as the end of their temporal unit

webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h

@@ -80,6 +80,8 @@ namespace jni
 			int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method);
 			void InvalidateCallback(JNIEnv * env);
 			void UpdateEncoderInfo(JNIEnv * env);
+			// Tells WebRTC that a frame given to the encoder will not come out.
+			void ReportDroppedFrame(uint32_t rtpTimestamp);
 
 			ScalingSettings GetScalingSettings(JNIEnv * env) const;
 			std::vector<ResolutionBitrateLimits> GetResolutionBitrateLimits(JNIEnv * env) const;

webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp

@@ -164,6 +164,9 @@ namespace jni
 		}
 
 		image.SetEncodedData(webrtc::EncodedImageBuffer::Create(output.data(), output.size()));
+		// A stream without layers: nothing else follows for this frame, which
+		// lets WebRTC release what it keeps of the frame until then.
+		image.set_end_of_temporal_unit(true);
 		image.set_frame_type(keyFrame
 			? webrtc::VideoFrameType::kVideoFrameKey
 			: webrtc::VideoFrameType::kVideoFrameDelta);

webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp

@@ -239,6 +239,8 @@ namespace jni
 		if (jFrame == nullptr) {
 			RTC_LOG(LS_WARNING) << "Java encoder: dropping a frame that cannot be converted to I420";
 
+			ReportDroppedFrame(frame.rtp_timestamp());
+
 			return WEBRTC_VIDEO_CODEC_OK;
 		}
 
@@ -268,10 +270,13 @@ namespace jni
 			result = WEBRTC_VIDEO_CODEC_ERROR;
 		}
 		else {
+			std::optional<uint32_t> givenUp;
+
 			{
 				webrtc::MutexLock lock(&frameExtraInfosLock);
 
 				if (frameExtraInfos.size() >= kMaxPendingFrames) {
+					givenUp = frameExtraInfos.front().rtpTimestamp;
 					frameExtraInfos.pop_front();
 				}
 
@@ -281,6 +286,10 @@ namespace jni
 				});
 			}
 
+			if (givenUp) {
+				ReportDroppedFrame(*givenUp);
+			}
+
 			jobject status = env->CallObjectMethod(encoder, javaClass->encode, jFrame, info);
 			result = ToNativeCodecStatus(env, status, "encode");
 		}
@@ -359,29 +368,40 @@ namespace jni
 		webrtc::EncodedImage image = EncodedImage::toNative(env, imageRef);
 		const int64_t captureTimeNs = EncodedImage::getCaptureTimeNs(env, imageRef);
 
-		FrameExtraInfo extraInfo;
+		std::optional<FrameExtraInfo> extraInfo;
+		std::vector<uint32_t> dropped;
 
 		{
 			webrtc::MutexLock lock(&frameExtraInfosLock);
 
 			// Frames come out in the order they went in, though an encoder
 			// may drop some, so the records of older frames are stale.
 			while (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs < captureTimeNs) {
+				dropped.push_back(frameExtraInfos.front().rtpTimestamp);
 				frameExtraInfos.pop_front();
 			}
 
-			if (frameExtraInfos.empty() || frameExtraInfos.front().captureTimeNs != captureTimeNs) {
-				RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time "
-					<< captureTimeNs << " ns";
-				return;
+			if (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs == captureTimeNs) {
+				extraInfo = frameExtraInfos.front();
+				frameExtraInfos.pop_front();
 			}
+		}
 
-			extraInfo = frameExtraInfos.front();
-			frameExtraInfos.pop_front();
+		for (uint32_t rtpTimestamp : dropped) {
+			ReportDroppedFrame(rtpTimestamp);
 		}
 
-		image.SetRtpTimestamp(extraInfo.rtpTimestamp);
+		if (!extraInfo) {
+			RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time "
+				<< captureTimeNs << " ns";
+			return;
+		}
+
+		image.SetRtpTimestamp(extraInfo->rtpTimestamp);
 		image.capture_time_ms_ = captureTimeNs / webrtc::kNumNanosecsPerMillisec;
+		// The Java encoders produce a single layer, so every frame ends its
+		// temporal unit.
+		image.set_end_of_temporal_unit(true);
 
 		if (image.qp_ < 0) {
 			image.qp_ = ParseQp(image);
@@ -395,6 +415,16 @@ namespace jni
 		}
 	}
 
+	void VideoEncoderWrapper::ReportDroppedFrame(uint32_t rtpTimestamp)
+	{
+		webrtc::EncodedImageCallback * encodeCallback = callback.load();
+
+		if (encodeCallback != nullptr) {
+			// A single layer, so nothing else follows for the frame.
+			encodeCallback->OnFrameDropped(rtpTimestamp, /*spatial_id=*/0, /*is_end_of_temporal_unit=*/true);
+		}
+	}
+
 	int32_t VideoEncoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method)
 	{
 		if (status >= 0) {

webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java

@@ -228,6 +228,26 @@ private void assertRoundTrip(VideoCodecInfo codec, boolean javaBuffers) throws E
 		assertTrue(decoderFactory.last.get().released.get(), "decoder was never released");
 	}
 
+	@Test
+	void javaEncoderDroppingFramesKeepsSending() throws Exception {
+		// A Java encoder drops a frame by producing nothing for it. WebRTC is
+		// told about the frames dropped once a later one comes out, and the
+		// frames encoded still arrive.
+		TestEncoderFactory encoderFactory = new TestEncoderFactory();
+		encoderFactory.dropEveryOtherFrame = true;
+
+		PeerConnectionFactory built = testCodecFactory(encoderFactory, new TestDecoderFactory());
+
+		try {
+			assertFramesReceived(built, TEST_CODEC);
+		}
+		finally {
+			built.dispose();
+		}
+
+		assertTrue(encoderFactory.dropped.get() > 0, "the encoder dropped no frames");
+	}
+
 	@Test
 	void factoryDelegatesToBuiltInCodec() throws Exception {
 		AtomicInteger encoders = new AtomicInteger();
@@ -499,10 +519,14 @@ static class TestEncoderFactory implements VideoEncoderFactory {
 		final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory();
 		final AtomicInteger created = new AtomicInteger();
 		final AtomicReference<TestEncoder> last = new AtomicReference<>();
+		final AtomicInteger dropped = new AtomicInteger();
 		final VideoCodecInfo codec;
 
 		volatile EncodeListener onEncode;
 
+		/** Makes the encoders produce nothing for every other frame. */
+		volatile boolean dropEveryOtherFrame;
+
 
 		TestEncoderFactory() {
 			this(new VideoCodecInfo(TEST_CODEC));
@@ -600,6 +624,8 @@ static class TestEncoder implements VideoEncoder {
 
 		private Callback callback;
 
+		private int frames;
+
 
 		TestEncoder(TestEncoderFactory factory, boolean h265) {
 			this.factory = factory;
@@ -621,6 +647,12 @@ public VideoCodecStatus release() {
 
 		@Override
 		public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) {
+			if (factory.dropEveryOtherFrame && frames++ % 2 == 1) {
+				// Dropped: no output, as a rate controlled encoder would do.
+				factory.dropped.incrementAndGet();
+				return VideoCodecStatus.OK;
+			}
+
 			int width = frame.buffer.getWidth();
 			int height = frame.buffer.getHeight();
 			int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF;