From f3e1ba38047783f7607c55496439ad71d1d9abbe Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Tue, 13 Oct 2026 23:24:51 +0200 Subject: [PATCH] feat: report receiver source changes and audio packet infos Adds RTCRtpReceiverObserver.onSourceChanged, an AudioTrackSink.onData variant with the RTP packets the audio came from (collected only for sinks that implement it), and RTCOfferOptions.restrictToStableOrHaveLocalOffer, all new in M155. --- docs/guide/audio/headless-audio.md | 23 +++ .../src/main/cpp/include/JNI_AudioTrack.h | 4 +- .../src/main/cpp/include/api/AudioTrackSink.h | 20 +- .../main/cpp/include/api/RTCOfferOptions.h | 1 + .../main/cpp/include/api/RTCRtpPacketInfo.h | 46 +++++ .../cpp/include/api/RTCRtpReceiverObserver.h | 2 + .../src/main/cpp/src/JNI_AudioTrack.cpp | 4 +- .../src/main/cpp/src/api/AudioTrackSink.cpp | 44 ++++- .../src/main/cpp/src/api/RTCOfferOptions.cpp | 5 + .../src/main/cpp/src/api/RTCRtpPacketInfo.cpp | 82 +++++++++ .../cpp/src/api/RTCRtpReceiverObserver.cpp | 15 ++ .../dev/onvoid/webrtc/RTCOfferOptions.java | 10 + .../dev/onvoid/webrtc/RTCRtpPacketInfo.java | 77 ++++++++ .../onvoid/webrtc/RTCRtpReceiverObserver.java | 21 ++- .../onvoid/webrtc/media/audio/AudioTrack.java | 19 +- .../webrtc/media/audio/AudioTrackSink.java | 22 +++ .../webrtc/RTCRtpReceiverSourceTests.java | 174 ++++++++++++++++++ 17 files changed, 555 insertions(+), 14 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h create mode 100644 webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java diff --git a/docs/guide/audio/headless-audio.md b/docs/guide/audio/headless-audio.md index aa044ed02..2c95351cd 100644 --- a/docs/guide/audio/headless-audio.md +++ b/docs/guide/audio/headless-audio.md @@ -54,6 +54,29 @@ finally { - If you only need the audio pipeline to be ready when remote audio arrives, you may delay playout initialization until after creating your RTCPeerConnection. ::: +### Who sent the received audio + +A sink can also learn which RTP packets the audio was assembled from: their SSRC, contributing sources (CSRCs) and audio level. It implements the second `onData` method of `AudioTrackSink`, and is then called with it instead of the first; the packets are collected only for such a sink: + +```java +AudioTrackSink sink = new AudioTrackSink() { + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + // Not called for this sink. + } + + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames, + RTCRtpPacketInfo[] packetInfos) { + for (RTCRtpPacketInfo info : packetInfos) { + System.out.println(info.ssrc + " level " + info.audioLevel); + } + } +}; +``` + +A receiver learns its sources as playout pulls the audio too, so `RTCRtpReceiver.getSynchronizationSources()`, `getContributingSources()` and `RTCRtpReceiverObserver.onSourceChanged` stay empty for audio while playout is not running. + ## Sending audio diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h index 6ef6bae46..ec9870fe5 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h @@ -26,10 +26,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack * Method: addSinkInternal - * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;)J + * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;Z)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInternal - (JNIEnv *, jobject, jobject); + (JNIEnv *, jobject, jobject, jboolean); /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack diff --git a/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h b/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h index fc65fe926..1a27aa1e6 100644 --- a/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h +++ b/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h @@ -21,32 +21,48 @@ #include "JavaRef.h" #include "api/media_stream_interface.h" +#include "api/rtp_packet_infos.h" #include +#include +#include namespace jni { class AudioTrackSink : public webrtc::AudioTrackSinkInterface { public: - AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink); + // With packetInfos, the Java sink is called with the RTP packets + // the audio was assembled from. + AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink, bool packetInfos); ~AudioTrackSink() = default; - // AudioTrackSinkInterface implementation. + // AudioTrackSinkInterface implementation. WebRTC calls the variant + // its source knows the most for. void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames) override; + void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs) override; + void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs, const webrtc::RtpPacketInfos & packetInfos) override; private: + void Deliver(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + const webrtc::RtpPacketInfos * infos); + class JavaAudioTrackSinkClass : public JavaClass { public: explicit JavaAudioTrackSinkClass(JNIEnv * env); jmethodID onData; + jmethodID onDataWithPacketInfos; }; private: JavaGlobalRef sink; + const bool packetInfos; + const std::shared_ptr javaClass; }; } diff --git a/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h b/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h index 0c2d87d0b..2afabc941 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h @@ -37,6 +37,7 @@ namespace jni jmethodID ctor; jfieldID vad; jfieldID iceRestart; + jfieldID restrictToStableOrHaveLocalOffer; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions & nativeType); diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h new file mode 100644 index 000000000..1f973c242 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_RTC_RTP_PACKET_INFO_H_ +#define JNI_WEBRTC_API_RTC_RTP_PACKET_INFO_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" + +#include + +namespace jni +{ + namespace RTCRtpPacketInfo + { + class JavaRTCRtpPacketInfoClass : public JavaClass + { + public: + explicit JavaRTCRtpPacketInfoClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + }; + + JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpPacketInfo & info); + JavaLocalRef toJavaArray(JNIEnv * env, const webrtc::RtpPacketInfos & infos); + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h index fb1dfe655..a91ade7a4 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h @@ -35,6 +35,7 @@ namespace jni // RtpReceiverObserverInterface implementation. void OnFirstPacketReceived(webrtc::MediaType media_type) override; + void OnSourceChanged(bool ssrc_changed, bool csrc_changed) override; private: class JavaRTCRtpReceiverObserverClass : public JavaClass @@ -43,6 +44,7 @@ namespace jni explicit JavaRTCRtpReceiverObserverClass(JNIEnv * env); jmethodID onFirstPacketReceived; + jmethodID onSourceChanged; }; private: diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp index 4469bb098..81c75c962 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp @@ -22,7 +22,7 @@ #include "api/media_stream_interface.h" JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInternal -(JNIEnv * env, jobject caller, jobject jsink) +(JNIEnv * env, jobject caller, jobject jsink, jboolean packetInfos) { if (jsink == nullptr) { env->Throw(jni::JavaNullPointerException(env, "AudioTrackSink must not be null")); @@ -32,7 +32,7 @@ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInt webrtc::AudioTrackInterface * track = GetHandle(env, caller); CHECK_HANDLEV(track, 0); - auto sink = new jni::AudioTrackSink(env, jni::JavaGlobalRef(env, jsink)); + auto sink = new jni::AudioTrackSink(env, jni::JavaGlobalRef(env, jsink), packetInfos); track->AddSink(sink); diff --git a/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp b/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp index f9451565e..d8c57c5d4 100644 --- a/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp +++ b/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp @@ -15,6 +15,7 @@ */ #include "api/AudioTrackSink.h" +#include "api/RTCRtpPacketInfo.h" #include "api/WebRTCUtils.h" #include "JavaClasses.h" #include "JavaUtils.h" @@ -22,13 +23,32 @@ namespace jni { - AudioTrackSink::AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink) : + AudioTrackSink::AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink, bool packetInfos) : sink(sink), + packetInfos(packetInfos), javaClass(JavaClasses::get(env)) { } void AudioTrackSink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames) + { + Deliver(data, bitsPerSample, sampleRate, channels, frames, nullptr); + } + + void AudioTrackSink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs) + { + Deliver(data, bitsPerSample, sampleRate, channels, frames, nullptr); + } + + void AudioTrackSink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs, const webrtc::RtpPacketInfos & packetInfos) + { + Deliver(data, bitsPerSample, sampleRate, channels, frames, &packetInfos); + } + + void AudioTrackSink::Deliver(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + const webrtc::RtpPacketInfos * infos) { JNIEnv * env = AttachCurrentThread(); @@ -43,8 +63,25 @@ namespace jni if (dataArray.get() != nullptr) { env->SetByteArrayRegion(dataArray.get(), 0, dataSize, buffer); - env->CallVoidMethod(sink, javaClass->onData, dataArray.get(), static_cast(bitsPerSample), - static_cast(sampleRate), static_cast(channels), static_cast(frames)); + + if (packetInfos) { + try { + JavaLocalRef infoArray = (infos != nullptr) + ? RTCRtpPacketInfo::toJavaArray(env, *infos) + : RTCRtpPacketInfo::toJavaArray(env, webrtc::RtpPacketInfos()); + + env->CallVoidMethod(sink, javaClass->onDataWithPacketInfos, dataArray.get(), + static_cast(bitsPerSample), static_cast(sampleRate), + static_cast(channels), static_cast(frames), infoArray.get()); + } + catch (...) { + ThrowCxxJavaException(env); + } + } + else { + env->CallVoidMethod(sink, javaClass->onData, dataArray.get(), static_cast(bitsPerSample), + static_cast(sampleRate), static_cast(channels), static_cast(frames)); + } } ReportPendingException(env); @@ -55,5 +92,6 @@ namespace jni jclass cls = FindClass(env, PKG_AUDIO"AudioTrackSink"); onData = GetMethod(env, cls, "onData", "([BIIII)V"); + onDataWithPacketInfos = GetMethod(env, cls, "onData", "([BIIII[L" PKG "RTCRtpPacketInfo;)V"); } } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp b/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp index d009a12b3..44ad398f1 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp @@ -34,6 +34,8 @@ namespace jni env->SetBooleanField(obj, javaClass->vad, nativeType.voice_activity_detection); env->SetBooleanField(obj, javaClass->iceRestart, nativeType.ice_restart); + env->SetBooleanField(obj, javaClass->restrictToStableOrHaveLocalOffer, + nativeType.restrict_offer_to_stable_or_have_local_offer); return JavaLocalRef(env, obj); } @@ -47,6 +49,8 @@ namespace jni webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; options.voice_activity_detection = obj.getBoolean(javaClass->vad); options.ice_restart = obj.getBoolean(javaClass->iceRestart); + options.restrict_offer_to_stable_or_have_local_offer = + obj.getBoolean(javaClass->restrictToStableOrHaveLocalOffer); return options; } @@ -59,6 +63,7 @@ namespace jni vad = GetFieldID(env, cls, "voiceActivityDetection", "Z"); iceRestart = GetFieldID(env, cls, "iceRestart", "Z"); + restrictToStableOrHaveLocalOffer = GetFieldID(env, cls, "restrictToStableOrHaveLocalOffer", "Z"); } } } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp new file mode 100644 index 000000000..9e0569b64 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/RTCRtpPacketInfo.h" +#include "JavaClasses.h" +#include "JavaPrimitive.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include + +namespace jni +{ + namespace RTCRtpPacketInfo + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpPacketInfo & info) + { + const auto javaClass = JavaClasses::get(env); + + const std::vector & csrcs = info.csrcs(); + std::vector sources(csrcs.begin(), csrcs.end()); + + JavaLocalRef jSources(env, env->NewLongArray(static_cast(sources.size()))); + ExceptionCheck(env); + + env->SetLongArrayRegion(jSources.get(), 0, static_cast(sources.size()), sources.data()); + + JavaLocalRef jAudioLevel(env, nullptr); + + if (info.audio_level().has_value()) { + jAudioLevel = Integer::create(env, static_cast(*info.audio_level())); + } + + jlong receiveTimeMs = info.receive_time().IsFinite() ? info.receive_time().ms() : 0; + + jobject object = env->NewObject(javaClass->cls, javaClass->ctor, + static_cast(info.ssrc()), jSources.get(), static_cast(info.rtp_timestamp()), + receiveTimeMs, jAudioLevel.get()); + ExceptionCheck(env); + + return JavaLocalRef(env, object); + } + + JavaLocalRef toJavaArray(JNIEnv * env, const webrtc::RtpPacketInfos & infos) + { + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef array(env, env->NewObjectArray(static_cast(infos.size()), javaClass->cls, nullptr)); + ExceptionCheck(env); + + jsize index = 0; + + for (const webrtc::RtpPacketInfo & info : infos) { + JavaLocalRef jInfo = toJava(env, info); + + env->SetObjectArrayElement(array.get(), index++, jInfo.get()); + } + + return array; + } + + JavaRTCRtpPacketInfoClass::JavaRTCRtpPacketInfoClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCRtpPacketInfo"); + + ctor = GetMethod(env, cls, "", "(J[JJJLjava/lang/Integer;)V"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp index d44b8e6a8..b40b6f797 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp @@ -49,10 +49,25 @@ namespace jni ReportPendingException(env); } + void RTCRtpReceiverObserver::OnSourceChanged(bool ssrc_changed, bool csrc_changed) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return; + } + + env->CallVoidMethod(observer, javaClass->onSourceChanged, + static_cast(ssrc_changed), static_cast(csrc_changed)); + + ReportPendingException(env); + } + RTCRtpReceiverObserver::JavaRTCRtpReceiverObserverClass::JavaRTCRtpReceiverObserverClass(JNIEnv * env) { jclass cls = FindClass(env, PKG"RTCRtpReceiverObserver"); onFirstPacketReceived = GetMethod(env, cls, "onFirstPacketReceived", "(L" PKG_MEDIA "MediaType;)V"); + onSourceChanged = GetMethod(env, cls, "onSourceChanged", "(ZZ)V"); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java index 7fc848068..89ba75364 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java @@ -35,6 +35,15 @@ public class RTCOfferOptions extends RTCOfferAnswerOptions { */ public boolean iceRestart; + /** + * When set to {@code true}, creating an offer fails unless the signaling + * state is {@link RTCSignalingState#STABLE} or + * {@link RTCSignalingState#HAVE_LOCAL_OFFER}, as the WebRTC specification's + * createOffer() requires. When {@code false}, an offer can be created in any + * state but closed, as JSEP allows. + */ + public boolean restrictToStableOrHaveLocalOffer; + /** * Creates an instance of RTCOfferOptions. @@ -43,6 +52,7 @@ public RTCOfferOptions() { super(); iceRestart = false; + restrictToStableOrHaveLocalOffer = false; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java new file mode 100644 index 000000000..145d7c12a --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import java.util.Arrays; + +/** + * What an RTP packet that some received audio was assembled from tells about + * it: who sent it, who contributed to it, and how loud it was. + * + * @author Alex Andres + * + * @see dev.onvoid.webrtc.media.audio.AudioTrackSink + */ +public class RTCRtpPacketInfo { + + /** The synchronization source (SSRC) of the packet. */ + public final long ssrc; + + /** The contributing sources (CSRCs) of the packet, possibly none. */ + public final long[] contributingSources; + + /** The RTP timestamp of the packet. */ + public final long rtpTimestamp; + + /** The local time the packet was received, in milliseconds. */ + public final long receiveTimeMs; + + /** + * The audio level of the packet in -dBov, from 0 (loudest) to 127 + * (silence), or null if the sender gave none. + */ + public final Integer audioLevel; + + + /** + * Creates an instance of RTCRtpPacketInfo. + * + * @param ssrc The synchronization source of the packet. + * @param contributingSources The contributing sources of the packet. + * @param rtpTimestamp The RTP timestamp of the packet. + * @param receiveTimeMs The local time the packet was received, in + * milliseconds. + * @param audioLevel The audio level of the packet in -dBov, or + * null. + */ + public RTCRtpPacketInfo(long ssrc, long[] contributingSources, long rtpTimestamp, long receiveTimeMs, + Integer audioLevel) { + this.ssrc = ssrc; + this.contributingSources = contributingSources; + this.rtpTimestamp = rtpTimestamp; + this.receiveTimeMs = receiveTimeMs; + this.audioLevel = audioLevel; + } + + @Override + public String toString() { + return String.format("%s@%d [ssrc=%d, contributingSources=%s, rtpTimestamp=%d, receiveTimeMs=%d, audioLevel=%s]", + RTCRtpPacketInfo.class.getSimpleName(), hashCode(), ssrc, Arrays.toString(contributingSources), + rtpTimestamp, receiveTimeMs, audioLevel); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java index 6f42cdff0..f24496446 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java @@ -20,10 +20,11 @@ /** * Callback interface to learn when an {@link RTCRtpReceiver} receives its - * first RTP packet, for example to show media only once it actually flows. + * first RTP packet, for example to show media only once it actually flows, + * and when the sources of its media change. *

- * The callback runs on the peer connection's signaling thread. An exception - * thrown by it is reported to the thread's uncaught exception handler. + * The callbacks run on the peer connection's signaling thread. An exception + * thrown by one is reported to the thread's uncaught exception handler. * * @author Alex Andres * @@ -39,4 +40,18 @@ public interface RTCRtpReceiverObserver { */ void onFirstPacketReceived(MediaType mediaType); + /** + * The synchronization or contributing sources of the received media have + * changed, so {@link RTCRtpReceiver#getSynchronizationSources()} or + * {@link RTCRtpReceiver#getContributingSources()} return something new. + * When the observer is set after sources are known, this is called right + * away. Does nothing unless implemented. + * + * @param ssrcChanged Whether the synchronization source has changed. + * @param csrcChanged Whether the set of contributing sources has changed. + */ + default void onSourceChanged(boolean ssrcChanged, boolean csrcChanged) { + + } + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java index 0fa70feeb..68a299450 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java @@ -19,6 +19,7 @@ import static java.util.Objects.isNull; import static java.util.Objects.nonNull; +import dev.onvoid.webrtc.RTCRtpPacketInfo; import dev.onvoid.webrtc.media.MediaStreamTrack; import java.util.IdentityHashMap; @@ -57,11 +58,25 @@ public void addSink(AudioTrackSink sink) { return; } - final long nativeSink = addSinkInternal(sink); + final long nativeSink = addSinkInternal(sink, takesPacketInfos(sink)); sinks.put(sink, nativeSink); } + /** + * Whether the sink implements the method that receives the RTP packets, + * which are only collected for a sink that does. + */ + private static boolean takesPacketInfos(AudioTrackSink sink) { + try { + return sink.getClass().getMethod("onData", byte[].class, int.class, int.class, int.class, + int.class, RTCRtpPacketInfo[].class).getDeclaringClass() != AudioTrackSink.class; + } + catch (NoSuchMethodException e) { + return false; + } + } + /** * Removes an AudioSink from the track. If the AudioSink was not attached to * the track, this is a no-op. @@ -106,7 +121,7 @@ public void setVolume(double volume) { private native void setVolumeInternal(double volume); - private native long addSinkInternal(AudioTrackSink sink); + private native long addSinkInternal(AudioTrackSink sink, boolean packetInfos); private native void removeSinkInternal(long sinkHandle); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java index bcfc57b68..a77cd4a0b 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java @@ -16,8 +16,30 @@ package dev.onvoid.webrtc.media.audio; +import dev.onvoid.webrtc.RTCRtpPacketInfo; + public interface AudioTrackSink { void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames); + /** + * Receives audio along with the RTP packets it was assembled from, which + * tell who sent and contributed to it and how loud each was. A sink that + * implements this method is called with it instead of the one without + * packets; the packets are only collected for such a sink. + *

+ * Audio of a local track has no packets, so the array is empty for it. + * + * @param data The audio samples. + * @param bitsPerSample The size of a sample in bits. + * @param sampleRate The sample rate in Hz. + * @param channels The number of channels. + * @param frames The number of frames, samples per channel. + * @param packetInfos The packets the audio was assembled from. + */ + default void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames, + RTCRtpPacketInfo[] packetInfos) { + onData(data, bitsPerSample, sampleRate, channels, frames); + } + } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java new file mode 100644 index 000000000..1d215affb --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java @@ -0,0 +1,174 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; +import dev.onvoid.webrtc.media.audio.HeadlessAudioDeviceModule; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests what a receiver tells about the sources of its media, and the offer + * option that keeps offers to the states the WebRTC specification allows. + */ +@Execution(ExecutionMode.SAME_THREAD) +class RTCRtpReceiverSourceTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + + @Test + void observerHearsSourceChange() throws Exception { + AtomicBoolean ssrcChanged = new AtomicBoolean(); + CountDownLatch changed = new CountDownLatch(1); + + // Sources are tracked as frames are delivered, which for audio needs + // playout pulling it; video frames are delivered as they are decoded. + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + receiver.setObserver(new RTCRtpReceiverObserver() { + @Override + public void onFirstPacketReceived(MediaType mediaType) { } + + @Override + public void onSourceChanged(boolean ssrc, boolean csrc) { + if (ssrc) { + ssrcChanged.set(true); + } + changed.countDown(); + } + }); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(changed.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "no source change reported"); + assertTrue(ssrcChanged.get(), "the first packet did not change the SSRC"); + + receiver.setObserver(null); + receiver.dispose(); + } + } + + @Test + void sinkReceivesPacketInfos() throws Exception { + // Remote audio reaches a sink only while playout pulls it, which the + // dummy module of the other tests does not do. + HeadlessAudioDeviceModule audioModule = new HeadlessAudioDeviceModule(); + PeerConnectionFactory headlessFactory = new PeerConnectionFactory(audioModule); + + audioModule.initPlayout(); + audioModule.startPlayout(); + + AtomicReference info = new AtomicReference<>(); + AtomicInteger withoutPackets = new AtomicInteger(); + CountDownLatch received = new CountDownLatch(10); + + try { + try (TestMediaCall call = new TestMediaCall(headlessFactory, false, true)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("audio"); + AudioTrack track = (AudioTrack) receiver.getTrack(); + AudioTrackSink sink = new AudioTrackSink() { + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + withoutPackets.incrementAndGet(); + } + + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames, + RTCRtpPacketInfo[] packetInfos) { + if (packetInfos.length > 0) { + info.compareAndSet(null, packetInfos[0]); + received.countDown(); + } + } + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "no audio with packets received"); + assertEquals(0, withoutPackets.get(), "the sink was also called without packets"); + + RTCRtpPacketInfo first = info.get(); + + assertTrue(first.ssrc > 0); + assertTrue(first.receiveTimeMs > 0); + assertNotNull(first.contributingSources); + + track.removeSink(sink); + receiver.dispose(); + } + } + finally { + audioModule.stopPlayout(); + headlessFactory.dispose(); + audioModule.dispose(); + } + } + + @Test + void offerCanBeRestrictedToStableState() throws Exception { + TestPeerConnection offerer = new TestPeerConnection(factory); + TestPeerConnection answerer = new TestPeerConnection(factory); + + try { + // The answerer now has a remote offer: JSEP lets it create an + // offer of its own, the WebRTC specification does not. + answerer.setRemoteDescription(offerer.createOffer()); + + RTCOfferOptions restrictedOptions = new RTCOfferOptions(); + restrictedOptions.restrictToStableOrHaveLocalOffer = true; + + TestCreateDescObserver restricted = new TestCreateDescObserver(); + answerer.getPeerConnection().createOffer(restrictedOptions, restricted); + + assertThrows(ExecutionException.class, () -> restricted.get(5, TimeUnit.SECONDS)); + + TestCreateDescObserver allowed = new TestCreateDescObserver(); + answerer.getPeerConnection().createOffer(new RTCOfferOptions(), allowed); + + assertNotNull(allowed.get(5, TimeUnit.SECONDS)); + } + finally { + offerer.close(); + answerer.close(); + } + } + +}