diff --git a/docs/guide/audio/headless-audio.md b/docs/guide/audio/headless-audio.md index aa044ed02..2c95351cd 100644 --- a/docs/guide/audio/headless-audio.md +++ b/docs/guide/audio/headless-audio.md @@ -54,6 +54,29 @@ finally { - If you only need the audio pipeline to be ready when remote audio arrives, you may delay playout initialization until after creating your RTCPeerConnection. ::: +### Who sent the received audio + +A sink can also learn which RTP packets the audio was assembled from: their SSRC, contributing sources (CSRCs) and audio level. It implements the second `onData` method of `AudioTrackSink`, and is then called with it instead of the first; the packets are collected only for such a sink: + +```java +AudioTrackSink sink = new AudioTrackSink() { + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + // Not called for this sink. + } + + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames, + RTCRtpPacketInfo[] packetInfos) { + for (RTCRtpPacketInfo info : packetInfos) { + System.out.println(info.ssrc + " level " + info.audioLevel); + } + } +}; +``` + +A receiver learns its sources as playout pulls the audio too, so `RTCRtpReceiver.getSynchronizationSources()`, `getContributingSources()` and `RTCRtpReceiverObserver.onSourceChanged` stay empty for audio while playout is not running. + ## Sending audio diff --git a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h index 6ef6bae46..ec9870fe5 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h +++ b/webrtc-jni/src/main/cpp/include/JNI_AudioTrack.h @@ -26,10 +26,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack * Method: addSinkInternal - * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;)J + * Signature: (Ldev/onvoid/webrtc/media/audio/AudioTrackSink;Z)J */ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInternal - (JNIEnv *, jobject, jobject); + (JNIEnv *, jobject, jobject, jboolean); /* * Class: dev_onvoid_webrtc_media_audio_AudioTrack diff --git a/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h b/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h index fc65fe926..1a27aa1e6 100644 --- a/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h +++ b/webrtc-jni/src/main/cpp/include/api/AudioTrackSink.h @@ -21,32 +21,48 @@ #include "JavaRef.h" #include "api/media_stream_interface.h" +#include "api/rtp_packet_infos.h" #include +#include +#include namespace jni { class AudioTrackSink : public webrtc::AudioTrackSinkInterface { public: - AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink); + // With packetInfos, the Java sink is called with the RTP packets + // the audio was assembled from. + AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink, bool packetInfos); ~AudioTrackSink() = default; - // AudioTrackSinkInterface implementation. + // AudioTrackSinkInterface implementation. WebRTC calls the variant + // its source knows the most for. void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames) override; + void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs) override; + void OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs, const webrtc::RtpPacketInfos & packetInfos) override; private: + void Deliver(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + const webrtc::RtpPacketInfos * infos); + class JavaAudioTrackSinkClass : public JavaClass { public: explicit JavaAudioTrackSinkClass(JNIEnv * env); jmethodID onData; + jmethodID onDataWithPacketInfos; }; private: JavaGlobalRef sink; + const bool packetInfos; + const std::shared_ptr javaClass; }; } diff --git a/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h b/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h index 0c2d87d0b..2afabc941 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCOfferOptions.h @@ -37,6 +37,7 @@ namespace jni jmethodID ctor; jfieldID vad; jfieldID iceRestart; + jfieldID restrictToStableOrHaveLocalOffer; }; JavaLocalRef toJava(JNIEnv * env, const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions & nativeType); diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h new file mode 100644 index 000000000..1f973c242 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpPacketInfo.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_RTC_RTP_PACKET_INFO_H_ +#define JNI_WEBRTC_API_RTC_RTP_PACKET_INFO_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" + +#include + +namespace jni +{ + namespace RTCRtpPacketInfo + { + class JavaRTCRtpPacketInfoClass : public JavaClass + { + public: + explicit JavaRTCRtpPacketInfoClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + }; + + JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpPacketInfo & info); + JavaLocalRef toJavaArray(JNIEnv * env, const webrtc::RtpPacketInfos & infos); + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h b/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h index fb1dfe655..a91ade7a4 100644 --- a/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h +++ b/webrtc-jni/src/main/cpp/include/api/RTCRtpReceiverObserver.h @@ -35,6 +35,7 @@ namespace jni // RtpReceiverObserverInterface implementation. void OnFirstPacketReceived(webrtc::MediaType media_type) override; + void OnSourceChanged(bool ssrc_changed, bool csrc_changed) override; private: class JavaRTCRtpReceiverObserverClass : public JavaClass @@ -43,6 +44,7 @@ namespace jni explicit JavaRTCRtpReceiverObserverClass(JNIEnv * env); jmethodID onFirstPacketReceived; + jmethodID onSourceChanged; }; private: diff --git a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp index 4469bb098..81c75c962 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_AudioTrack.cpp @@ -22,7 +22,7 @@ #include "api/media_stream_interface.h" JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInternal -(JNIEnv * env, jobject caller, jobject jsink) +(JNIEnv * env, jobject caller, jobject jsink, jboolean packetInfos) { if (jsink == nullptr) { env->Throw(jni::JavaNullPointerException(env, "AudioTrackSink must not be null")); @@ -32,7 +32,7 @@ JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_audio_AudioTrack_addSinkInt webrtc::AudioTrackInterface * track = GetHandle(env, caller); CHECK_HANDLEV(track, 0); - auto sink = new jni::AudioTrackSink(env, jni::JavaGlobalRef(env, jsink)); + auto sink = new jni::AudioTrackSink(env, jni::JavaGlobalRef(env, jsink), packetInfos); track->AddSink(sink); diff --git a/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp b/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp index f9451565e..d8c57c5d4 100644 --- a/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp +++ b/webrtc-jni/src/main/cpp/src/api/AudioTrackSink.cpp @@ -15,6 +15,7 @@ */ #include "api/AudioTrackSink.h" +#include "api/RTCRtpPacketInfo.h" #include "api/WebRTCUtils.h" #include "JavaClasses.h" #include "JavaUtils.h" @@ -22,13 +23,32 @@ namespace jni { - AudioTrackSink::AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink) : + AudioTrackSink::AudioTrackSink(JNIEnv * env, const JavaGlobalRef & sink, bool packetInfos) : sink(sink), + packetInfos(packetInfos), javaClass(JavaClasses::get(env)) { } void AudioTrackSink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames) + { + Deliver(data, bitsPerSample, sampleRate, channels, frames, nullptr); + } + + void AudioTrackSink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs) + { + Deliver(data, bitsPerSample, sampleRate, channels, frames, nullptr); + } + + void AudioTrackSink::OnData(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + std::optional absoluteCaptureTimestampMs, const webrtc::RtpPacketInfos & packetInfos) + { + Deliver(data, bitsPerSample, sampleRate, channels, frames, &packetInfos); + } + + void AudioTrackSink::Deliver(const void * data, int bitsPerSample, int sampleRate, size_t channels, size_t frames, + const webrtc::RtpPacketInfos * infos) { JNIEnv * env = AttachCurrentThread(); @@ -43,8 +63,25 @@ namespace jni if (dataArray.get() != nullptr) { env->SetByteArrayRegion(dataArray.get(), 0, dataSize, buffer); - env->CallVoidMethod(sink, javaClass->onData, dataArray.get(), static_cast(bitsPerSample), - static_cast(sampleRate), static_cast(channels), static_cast(frames)); + + if (packetInfos) { + try { + JavaLocalRef infoArray = (infos != nullptr) + ? RTCRtpPacketInfo::toJavaArray(env, *infos) + : RTCRtpPacketInfo::toJavaArray(env, webrtc::RtpPacketInfos()); + + env->CallVoidMethod(sink, javaClass->onDataWithPacketInfos, dataArray.get(), + static_cast(bitsPerSample), static_cast(sampleRate), + static_cast(channels), static_cast(frames), infoArray.get()); + } + catch (...) { + ThrowCxxJavaException(env); + } + } + else { + env->CallVoidMethod(sink, javaClass->onData, dataArray.get(), static_cast(bitsPerSample), + static_cast(sampleRate), static_cast(channels), static_cast(frames)); + } } ReportPendingException(env); @@ -55,5 +92,6 @@ namespace jni jclass cls = FindClass(env, PKG_AUDIO"AudioTrackSink"); onData = GetMethod(env, cls, "onData", "([BIIII)V"); + onDataWithPacketInfos = GetMethod(env, cls, "onData", "([BIIII[L" PKG "RTCRtpPacketInfo;)V"); } } diff --git a/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp b/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp index d009a12b3..44ad398f1 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCOfferOptions.cpp @@ -34,6 +34,8 @@ namespace jni env->SetBooleanField(obj, javaClass->vad, nativeType.voice_activity_detection); env->SetBooleanField(obj, javaClass->iceRestart, nativeType.ice_restart); + env->SetBooleanField(obj, javaClass->restrictToStableOrHaveLocalOffer, + nativeType.restrict_offer_to_stable_or_have_local_offer); return JavaLocalRef(env, obj); } @@ -47,6 +49,8 @@ namespace jni webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; options.voice_activity_detection = obj.getBoolean(javaClass->vad); options.ice_restart = obj.getBoolean(javaClass->iceRestart); + options.restrict_offer_to_stable_or_have_local_offer = + obj.getBoolean(javaClass->restrictToStableOrHaveLocalOffer); return options; } @@ -59,6 +63,7 @@ namespace jni vad = GetFieldID(env, cls, "voiceActivityDetection", "Z"); iceRestart = GetFieldID(env, cls, "iceRestart", "Z"); + restrictToStableOrHaveLocalOffer = GetFieldID(env, cls, "restrictToStableOrHaveLocalOffer", "Z"); } } } \ No newline at end of file diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp new file mode 100644 index 000000000..9e0569b64 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpPacketInfo.cpp @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/RTCRtpPacketInfo.h" +#include "JavaClasses.h" +#include "JavaPrimitive.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include + +namespace jni +{ + namespace RTCRtpPacketInfo + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::RtpPacketInfo & info) + { + const auto javaClass = JavaClasses::get(env); + + const std::vector & csrcs = info.csrcs(); + std::vector sources(csrcs.begin(), csrcs.end()); + + JavaLocalRef jSources(env, env->NewLongArray(static_cast(sources.size()))); + ExceptionCheck(env); + + env->SetLongArrayRegion(jSources.get(), 0, static_cast(sources.size()), sources.data()); + + JavaLocalRef jAudioLevel(env, nullptr); + + if (info.audio_level().has_value()) { + jAudioLevel = Integer::create(env, static_cast(*info.audio_level())); + } + + jlong receiveTimeMs = info.receive_time().IsFinite() ? info.receive_time().ms() : 0; + + jobject object = env->NewObject(javaClass->cls, javaClass->ctor, + static_cast(info.ssrc()), jSources.get(), static_cast(info.rtp_timestamp()), + receiveTimeMs, jAudioLevel.get()); + ExceptionCheck(env); + + return JavaLocalRef(env, object); + } + + JavaLocalRef toJavaArray(JNIEnv * env, const webrtc::RtpPacketInfos & infos) + { + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef array(env, env->NewObjectArray(static_cast(infos.size()), javaClass->cls, nullptr)); + ExceptionCheck(env); + + jsize index = 0; + + for (const webrtc::RtpPacketInfo & info : infos) { + JavaLocalRef jInfo = toJava(env, info); + + env->SetObjectArrayElement(array.get(), index++, jInfo.get()); + } + + return array; + } + + JavaRTCRtpPacketInfoClass::JavaRTCRtpPacketInfoClass(JNIEnv * env) + { + cls = FindClass(env, PKG"RTCRtpPacketInfo"); + + ctor = GetMethod(env, cls, "", "(J[JJJLjava/lang/Integer;)V"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp b/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp index d44b8e6a8..b40b6f797 100644 --- a/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp +++ b/webrtc-jni/src/main/cpp/src/api/RTCRtpReceiverObserver.cpp @@ -49,10 +49,25 @@ namespace jni ReportPendingException(env); } + void RTCRtpReceiverObserver::OnSourceChanged(bool ssrc_changed, bool csrc_changed) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return; + } + + env->CallVoidMethod(observer, javaClass->onSourceChanged, + static_cast(ssrc_changed), static_cast(csrc_changed)); + + ReportPendingException(env); + } + RTCRtpReceiverObserver::JavaRTCRtpReceiverObserverClass::JavaRTCRtpReceiverObserverClass(JNIEnv * env) { jclass cls = FindClass(env, PKG"RTCRtpReceiverObserver"); onFirstPacketReceived = GetMethod(env, cls, "onFirstPacketReceived", "(L" PKG_MEDIA "MediaType;)V"); + onSourceChanged = GetMethod(env, cls, "onSourceChanged", "(ZZ)V"); } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java index 7fc848068..89ba75364 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCOfferOptions.java @@ -35,6 +35,15 @@ public class RTCOfferOptions extends RTCOfferAnswerOptions { */ public boolean iceRestart; + /** + * When set to {@code true}, creating an offer fails unless the signaling + * state is {@link RTCSignalingState#STABLE} or + * {@link RTCSignalingState#HAVE_LOCAL_OFFER}, as the WebRTC specification's + * createOffer() requires. When {@code false}, an offer can be created in any + * state but closed, as JSEP allows. + */ + public boolean restrictToStableOrHaveLocalOffer; + /** * Creates an instance of RTCOfferOptions. @@ -43,6 +52,7 @@ public RTCOfferOptions() { super(); iceRestart = false; + restrictToStableOrHaveLocalOffer = false; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java new file mode 100644 index 000000000..145d7c12a --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpPacketInfo.java @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import java.util.Arrays; + +/** + * What an RTP packet that some received audio was assembled from tells about + * it: who sent it, who contributed to it, and how loud it was. + * + * @author Alex Andres + * + * @see dev.onvoid.webrtc.media.audio.AudioTrackSink + */ +public class RTCRtpPacketInfo { + + /** The synchronization source (SSRC) of the packet. */ + public final long ssrc; + + /** The contributing sources (CSRCs) of the packet, possibly none. */ + public final long[] contributingSources; + + /** The RTP timestamp of the packet. */ + public final long rtpTimestamp; + + /** The local time the packet was received, in milliseconds. */ + public final long receiveTimeMs; + + /** + * The audio level of the packet in -dBov, from 0 (loudest) to 127 + * (silence), or null if the sender gave none. + */ + public final Integer audioLevel; + + + /** + * Creates an instance of RTCRtpPacketInfo. + * + * @param ssrc The synchronization source of the packet. + * @param contributingSources The contributing sources of the packet. + * @param rtpTimestamp The RTP timestamp of the packet. + * @param receiveTimeMs The local time the packet was received, in + * milliseconds. + * @param audioLevel The audio level of the packet in -dBov, or + * null. + */ + public RTCRtpPacketInfo(long ssrc, long[] contributingSources, long rtpTimestamp, long receiveTimeMs, + Integer audioLevel) { + this.ssrc = ssrc; + this.contributingSources = contributingSources; + this.rtpTimestamp = rtpTimestamp; + this.receiveTimeMs = receiveTimeMs; + this.audioLevel = audioLevel; + } + + @Override + public String toString() { + return String.format("%s@%d [ssrc=%d, contributingSources=%s, rtpTimestamp=%d, receiveTimeMs=%d, audioLevel=%s]", + RTCRtpPacketInfo.class.getSimpleName(), hashCode(), ssrc, Arrays.toString(contributingSources), + rtpTimestamp, receiveTimeMs, audioLevel); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java index 6f42cdff0..f24496446 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/RTCRtpReceiverObserver.java @@ -20,10 +20,11 @@ /** * Callback interface to learn when an {@link RTCRtpReceiver} receives its - * first RTP packet, for example to show media only once it actually flows. + * first RTP packet, for example to show media only once it actually flows, + * and when the sources of its media change. *

- * The callback runs on the peer connection's signaling thread. An exception - * thrown by it is reported to the thread's uncaught exception handler. + * The callbacks run on the peer connection's signaling thread. An exception + * thrown by one is reported to the thread's uncaught exception handler. * * @author Alex Andres * @@ -39,4 +40,18 @@ public interface RTCRtpReceiverObserver { */ void onFirstPacketReceived(MediaType mediaType); + /** + * The synchronization or contributing sources of the received media have + * changed, so {@link RTCRtpReceiver#getSynchronizationSources()} or + * {@link RTCRtpReceiver#getContributingSources()} return something new. + * When the observer is set after sources are known, this is called right + * away. Does nothing unless implemented. + * + * @param ssrcChanged Whether the synchronization source has changed. + * @param csrcChanged Whether the set of contributing sources has changed. + */ + default void onSourceChanged(boolean ssrcChanged, boolean csrcChanged) { + + } + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java index 0fa70feeb..68a299450 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrack.java @@ -19,6 +19,7 @@ import static java.util.Objects.isNull; import static java.util.Objects.nonNull; +import dev.onvoid.webrtc.RTCRtpPacketInfo; import dev.onvoid.webrtc.media.MediaStreamTrack; import java.util.IdentityHashMap; @@ -57,11 +58,25 @@ public void addSink(AudioTrackSink sink) { return; } - final long nativeSink = addSinkInternal(sink); + final long nativeSink = addSinkInternal(sink, takesPacketInfos(sink)); sinks.put(sink, nativeSink); } + /** + * Whether the sink implements the method that receives the RTP packets, + * which are only collected for a sink that does. + */ + private static boolean takesPacketInfos(AudioTrackSink sink) { + try { + return sink.getClass().getMethod("onData", byte[].class, int.class, int.class, int.class, + int.class, RTCRtpPacketInfo[].class).getDeclaringClass() != AudioTrackSink.class; + } + catch (NoSuchMethodException e) { + return false; + } + } + /** * Removes an AudioSink from the track. If the AudioSink was not attached to * the track, this is a no-op. @@ -106,7 +121,7 @@ public void setVolume(double volume) { private native void setVolumeInternal(double volume); - private native long addSinkInternal(AudioTrackSink sink); + private native long addSinkInternal(AudioTrackSink sink, boolean packetInfos); private native void removeSinkInternal(long sinkHandle); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java index bcfc57b68..a77cd4a0b 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/AudioTrackSink.java @@ -16,8 +16,30 @@ package dev.onvoid.webrtc.media.audio; +import dev.onvoid.webrtc.RTCRtpPacketInfo; + public interface AudioTrackSink { void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames); + /** + * Receives audio along with the RTP packets it was assembled from, which + * tell who sent and contributed to it and how loud each was. A sink that + * implements this method is called with it instead of the one without + * packets; the packets are only collected for such a sink. + *

+ * Audio of a local track has no packets, so the array is empty for it. + * + * @param data The audio samples. + * @param bitsPerSample The size of a sample in bits. + * @param sampleRate The sample rate in Hz. + * @param channels The number of channels. + * @param frames The number of frames, samples per channel. + * @param packetInfos The packets the audio was assembled from. + */ + default void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames, + RTCRtpPacketInfo[] packetInfos) { + onData(data, bitsPerSample, sampleRate, channels, frames); + } + } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java new file mode 100644 index 000000000..1d215affb --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/RTCRtpReceiverSourceTests.java @@ -0,0 +1,174 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; +import dev.onvoid.webrtc.media.audio.HeadlessAudioDeviceModule; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests what a receiver tells about the sources of its media, and the offer + * option that keeps offers to the states the WebRTC specification allows. + */ +@Execution(ExecutionMode.SAME_THREAD) +class RTCRtpReceiverSourceTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + + @Test + void observerHearsSourceChange() throws Exception { + AtomicBoolean ssrcChanged = new AtomicBoolean(); + CountDownLatch changed = new CountDownLatch(1); + + // Sources are tracked as frames are delivered, which for audio needs + // playout pulling it; video frames are delivered as they are decoded. + try (TestMediaCall call = new TestMediaCall(factory, true, false)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + receiver.setObserver(new RTCRtpReceiverObserver() { + @Override + public void onFirstPacketReceived(MediaType mediaType) { } + + @Override + public void onSourceChanged(boolean ssrc, boolean csrc) { + if (ssrc) { + ssrcChanged.set(true); + } + changed.countDown(); + } + }); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(changed.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "no source change reported"); + assertTrue(ssrcChanged.get(), "the first packet did not change the SSRC"); + + receiver.setObserver(null); + receiver.dispose(); + } + } + + @Test + void sinkReceivesPacketInfos() throws Exception { + // Remote audio reaches a sink only while playout pulls it, which the + // dummy module of the other tests does not do. + HeadlessAudioDeviceModule audioModule = new HeadlessAudioDeviceModule(); + PeerConnectionFactory headlessFactory = new PeerConnectionFactory(audioModule); + + audioModule.initPlayout(); + audioModule.startPlayout(); + + AtomicReference info = new AtomicReference<>(); + AtomicInteger withoutPackets = new AtomicInteger(); + CountDownLatch received = new CountDownLatch(10); + + try { + try (TestMediaCall call = new TestMediaCall(headlessFactory, false, true)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("audio"); + AudioTrack track = (AudioTrack) receiver.getTrack(); + AudioTrackSink sink = new AudioTrackSink() { + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + withoutPackets.incrementAndGet(); + } + + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames, + RTCRtpPacketInfo[] packetInfos) { + if (packetInfos.length > 0) { + info.compareAndSet(null, packetInfos[0]); + received.countDown(); + } + } + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "no audio with packets received"); + assertEquals(0, withoutPackets.get(), "the sink was also called without packets"); + + RTCRtpPacketInfo first = info.get(); + + assertTrue(first.ssrc > 0); + assertTrue(first.receiveTimeMs > 0); + assertNotNull(first.contributingSources); + + track.removeSink(sink); + receiver.dispose(); + } + } + finally { + audioModule.stopPlayout(); + headlessFactory.dispose(); + audioModule.dispose(); + } + } + + @Test + void offerCanBeRestrictedToStableState() throws Exception { + TestPeerConnection offerer = new TestPeerConnection(factory); + TestPeerConnection answerer = new TestPeerConnection(factory); + + try { + // The answerer now has a remote offer: JSEP lets it create an + // offer of its own, the WebRTC specification does not. + answerer.setRemoteDescription(offerer.createOffer()); + + RTCOfferOptions restrictedOptions = new RTCOfferOptions(); + restrictedOptions.restrictToStableOrHaveLocalOffer = true; + + TestCreateDescObserver restricted = new TestCreateDescObserver(); + answerer.getPeerConnection().createOffer(restrictedOptions, restricted); + + assertThrows(ExecutionException.class, () -> restricted.get(5, TimeUnit.SECONDS)); + + TestCreateDescObserver allowed = new TestCreateDescObserver(); + answerer.getPeerConnection().createOffer(new RTCOfferOptions(), allowed); + + assertNotNull(allowed.get(5, TimeUnit.SECONDS)); + } + finally { + offerer.close(); + answerer.close(); + } + } + +}