From 9a91938ec66bb0b2253268da575256d9186d87cd Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 14 Oct 2026 10:09:38 +0200 Subject: [PATCH 1/3] Teams meetings: mark the newest speaker, and log more of a nameless participant The speaker history's first entry marked whoever spoke long ago; take the last. Log the history as it comes, and a nameless participant's devices. --- docs/research/teams-calls.md | 4 ++-- src/teams/calling/call.rs | 17 ++++++++++++++--- src/teams/calling/channel.rs | 2 +- src/teams/calling/types.rs | 4 ++++ 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/research/teams-calls.md b/docs/research/teams-calls.md index f278f024..22f04c02 100644 --- a/docs/research/teams-calls.md +++ b/docs/research/teams-calls.md @@ -1612,7 +1612,7 @@ last), the meeting took them (`sendonly`) but then neither answered our sent its acceptance twice. Without them, both worked. Until a recording shows what else the web client does, we offer the one camera line. It follows whoever speaks with a camera on, by the data channel's `dsh` -(the speakers' audio source ids, newest first), matched to the roster's +(the speakers' audio source ids, the newest last), matched to the roster's `main-audio` `sourceId`. **Several cameras, recorded (`teams.live.com10.har` and its data channel @@ -1639,7 +1639,7 @@ for one camera on each. **Who speaks.** The meeting mixes everyone's sound into one stream, with no audio level header, so loudness alone says that *someone* speaks. The -data channel's `dsh` says who spoke last (newest first, by audio source +data channel's `dsh` says who spoke (a history, the newest last, by audio source id). The bar and the tiles mark that person as speaking while the mixed sound is loud, as Teams' own speaking ring roughly does. diff --git a/src/teams/calling/call.rs b/src/teams/calling/call.rs index 210d882d..3c676d9d 100644 --- a/src/teams/calling/call.rs +++ b/src/teams/calling/call.rs @@ -189,11 +189,17 @@ impl People { && !self.by_mri.contains_key(mri) && let Some(details) = &them.details { + let devices: Vec = them + .endpoints + .values() + .map(|e| serde_json::Value::Object(e.rest.clone().into_iter().collect())) + .collect(); log::info!( - "Teams meeting: no name for {mri} ({} participant; role {:?}; details also say {})", + "Teams meeting: no name for {mri} ({} participant; role {:?}; details also say {}; its devices {})", details.participant_type.as_deref().unwrap_or("untyped"), them.role, serde_json::Value::Object(details.rest.clone().into_iter().collect()), + serde_json::Value::Array(devices), ); } let attendee = Attendee { @@ -1576,8 +1582,9 @@ impl Call { let Some(meeting) = &mut self.meeting else { return; }; - // The newest first; ourselves (not in the list) aside. - let speaker = history.iter().find_map(|&source| { + // The newest last (seen: the first marked whoever spoke + // long ago); ourselves (not in the list) aside. + let speaker = history.iter().rev().find_map(|&source| { meeting .people .list() @@ -1585,6 +1592,10 @@ impl Call { .find(|a| a.audio == Some(source)) .map(|a| a.mri) }); + log::info!( + "Teams meeting: spoke, by audio source: {history:?}; marking {}", + speaker.as_deref().unwrap_or("no one") + ); if speaker.is_some() && speaker != meeting.speaker { meeting.speaker = speaker.clone(); if let Some(mri) = speaker { diff --git a/src/teams/calling/channel.rs b/src/teams/calling/channel.rs index 87baac22..6ace61f8 100644 --- a/src/teams/calling/channel.rs +++ b/src/teams/calling/channel.rs @@ -145,7 +145,7 @@ pub enum Heard { Ready, /// A source request of ours was answered. SourceAnswered { sequence: u64, ok: bool }, - /// Who spoke last, by their audio's source id, newest first. + /// Who spoke, by their audio's source id: a history, the newest last. Speakers(Vec), } diff --git a/src/teams/calling/types.rs b/src/teams/calling/types.rs index 3b4e2a87..c62aff37 100644 --- a/src/teams/calling/types.rs +++ b/src/teams/calling/types.rs @@ -625,6 +625,10 @@ pub struct RosterEndpoint { /// in, `Call` for one never kept waiting (recorded). #[serde(deserialize_with = "nullable")] pub modality_joined: String, + /// Whatever else the roster says of the device, kept so a participant + /// without a name can be logged in full and the name found. + #[serde(flatten)] + pub rest: BTreeMap, } impl RosterEndpoint { From 1e61470cd12cbbeec81e1a821329c2d0c14ed028 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 14 Oct 2026 10:17:27 +0200 Subject: [PATCH 2/3] Teams meetings: a bot hidden by its device's metadata stays out of the room --- src/teams/calling/call.rs | 16 +++++++++++++++- src/teams/calling/types.rs | 14 +++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/teams/calling/call.rs b/src/teams/calling/call.rs index 3c676d9d..40e7f547 100644 --- a/src/teams/calling/call.rs +++ b/src/teams/calling/call.rs @@ -2124,7 +2124,21 @@ mod tests { let details = bot.details.as_mut().expect("details"); details.hidden = true; details.display_name = None; - update.participants.insert("28:recorder".into(), bot); + update + .participants + .insert("28:recorder".into(), bot.clone()); + // The recorder says so on its device instead (recorded). + let mut transcriber = bot; + transcriber.details.as_mut().expect("details").hidden = false; + for endpoint in transcriber.endpoints.values_mut() { + endpoint.rest.insert( + "endpointMetadata".into(), + serde_json::json!({"__platform": {"ui": {"hidden": true}}}), + ); + } + update + .participants + .insert("28:transcriber".into(), transcriber); let mut people = People::default(); assert!(!people.take(&update, "8:live:organizer")); assert!(people.list().is_empty()); diff --git a/src/teams/calling/types.rs b/src/teams/calling/types.rs index c62aff37..b917bda8 100644 --- a/src/teams/calling/types.rs +++ b/src/teams/calling/types.rs @@ -525,9 +525,12 @@ impl RosterParticipant { /// Whether the client would not show them: a hidden participant (a /// recording or transcription bot, say) is in the roster but not in - /// the room. + /// the room. The details may say so; a bot's device says so in its + /// metadata (recorded: `endpointMetadata.__platform.ui.hidden` on the + /// call recorder and on the transcription bot). pub fn is_hidden(&self) -> bool { self.details.as_ref().is_some_and(|d| d.hidden) + || self.endpoints.values().any(RosterEndpoint::is_hidden) } /// Whether the participant is muted: an endpoint of theirs says so. @@ -632,6 +635,15 @@ pub struct RosterEndpoint { } impl RosterEndpoint { + /// Whether the client hides this device (a bot's, recorded). + pub fn is_hidden(&self) -> bool { + self.rest + .get("endpointMetadata") + .and_then(|m| m.pointer("/__platform/ui/hidden")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + } + /// Whether this device waits in the lobby: it is there, it has never /// been in the call, and it is not in it now. One let in and gone /// again can still show its lobby media. From 89cb4e6bd6da3c6dbd4b4d98c92425129d218643 Mon Sep 17 00:00:00 2001 From: Yannick de Jong Date: Wed, 14 Oct 2026 10:22:33 +0200 Subject: [PATCH 3/3] Teams meetings: look for the live speaker in RTCP and in csrcInfo pushes The data channel's speaker history came once, on joining, and never again. The audio line offers dsh as an RTCP application message too, and the call registers a csrcInfo callback: both reach the call now, logged, and a JSON dsh among them counts. --- docs/research/teams-calls.md | 11 +++- src/teams/calling/call.rs | 119 ++++++++++++++++++++++++++++------- src/teams/calling/media.rs | 15 +++++ src/teams/calling/types.rs | 4 ++ 4 files changed, 122 insertions(+), 27 deletions(-) diff --git a/docs/research/teams-calls.md b/docs/research/teams-calls.md index 22f04c02..ecaa0a7a 100644 --- a/docs/research/teams-calls.md +++ b/docs/research/teams-calls.md @@ -1639,9 +1639,14 @@ for one camera on each. **Who speaks.** The meeting mixes everyone's sound into one stream, with no audio level header, so loudness alone says that *someone* speaks. The -data channel's `dsh` says who spoke (a history, the newest last, by audio source -id). The bar and the tiles mark that person as speaking while the mixed -sound is loud, as Teams' own speaking ring roughly does. +data channel's `dsh` says who spoke (a history, the newest last, by audio +source id), but it came **once**, right after the `ack`, and never again +in a meeting of two minutes with people talking: a snapshot on joining. +The audio line's SDP offers `dsh` as an RTCP application message too +(`a=rtcp-fb:* x-message app send:dsh,x-gain recv:dsh,x-gain`), and the +call registers a `csrcInfo` callback for the media controller; both are +now logged, to find the live signal. The bar and the tiles mark the last +speaker known as speaking while the mixed sound is loud. ### H.10 Sharing a screen in a meeting (recorded: `teams.live.com11.har` and its data channel log) diff --git a/src/teams/calling/call.rs b/src/teams/calling/call.rs index 40e7f547..daea8289 100644 --- a/src/teams/calling/call.rs +++ b/src/teams/calling/call.rs @@ -306,6 +306,7 @@ async fn run_meeting( watching_share: None, syns: 0, heard: 0, + feedback: 0, sharing: false, endpoint: endpoint.clone(), own_camera: None, @@ -576,6 +577,8 @@ struct InMeeting { syns: u32, /// How many messages came on it since it opened. heard: u32, + /// How many application messages came in RTCP. + feedback: u32, /// Whether we share our screen, as the meeting was last told. sharing: bool, /// Our endpoint id, by which the roster lists our own streams. @@ -897,6 +900,11 @@ impl Call { } } Some(MediaEvent::ChannelData(message)) => self.on_channel(&message, session, tell), + Some(MediaEvent::Feedback { + sender, + media, + payload, + }) => self.on_feedback(sender, media, &payload, session, tell), Some(MediaEvent::AudioFlowing) => { log::info!("Teams call: audio flows both ways"); tell(CallEvent::AudioFlowing); @@ -1366,6 +1374,11 @@ impl Call { self.offer_share(local).await; } } + Push::CsrcInfo(body) => { + let text = body.to_string(); + let cut = text.char_indices().nth(2000).map_or(text.len(), |(i, _)| i); + log::info!("Teams meeting: csrcInfo: {}", &text[..cut]); + } Push::Other(name) => log::debug!("Teams call: push {name} not acted on"), } None @@ -1579,35 +1592,93 @@ impl Call { } } super::channel::Heard::Speakers(history) => { - let Some(meeting) = &mut self.meeting else { - return; - }; - // The newest last (seen: the first marked whoever spoke - // long ago); ourselves (not in the list) aside. - let speaker = history.iter().rev().find_map(|&source| { - meeting - .people - .list() - .into_iter() - .find(|a| a.audio == Some(source)) - .map(|a| a.mri) - }); - log::info!( - "Teams meeting: spoke, by audio source: {history:?}; marking {}", - speaker.as_deref().unwrap_or("no one") - ); - if speaker.is_some() && speaker != meeting.speaker { - meeting.speaker = speaker.clone(); - if let Some(mri) = speaker { - tell(CallEvent::Speaker(mri)); - } - self.ask_for_video(session, tell); - } + self.spoke(&history, "the data channel", session, tell); } } } } + /// Who spoke, as a speaker history (`dsh`) from `way` says: the + /// newest last (seen: the first marked whoever spoke long ago); + /// ourselves (not in the list) aside. + fn spoke( + &mut self, + history: &[i64], + way: &str, + session: &MediaSession, + tell: &(impl Fn(CallEvent) + Send), + ) { + let Some(meeting) = &mut self.meeting else { + return; + }; + let speaker = history.iter().rev().find_map(|&source| { + meeting + .people + .list() + .into_iter() + .find(|a| a.audio == Some(source)) + .map(|a| a.mri) + }); + log::info!( + "Teams meeting: spoke, by audio source ({way}): {history:?}; marking {}", + speaker.as_deref().unwrap_or("no one") + ); + if speaker.is_some() && speaker != meeting.speaker { + meeting.speaker = speaker.clone(); + if let Some(mri) = speaker { + tell(CallEvent::Speaker(mri)); + } + self.ask_for_video(session, tell); + } + } + + /// An application message in RTCP: the audio line offers `dsh` this + /// way too. Its shape is not recorded, so the first few are logged + /// whole; one with a JSON `dsh` in it counts as a speaker history. + fn on_feedback( + &mut self, + sender: u32, + media: u32, + payload: &[u8], + session: &MediaSession, + tell: &(impl Fn(CallEvent) + Send), + ) { + let Some(meeting) = &mut self.meeting else { + return; + }; + meeting.feedback += 1; + if meeting.feedback <= 5 || meeting.feedback % 100 == 0 { + let head: Vec = payload + .iter() + .take(32) + .map(|b| format!("{b:02x}")) + .collect(); + log::info!( + "Teams meeting: RTCP application message {} from {sender} about {media}, {} bytes: {} / {:?}", + meeting.feedback, + payload.len(), + head.join(" "), + String::from_utf8_lossy(payload) + ); + } + // JSON, if any, from its first brace. + let Some(start) = payload.iter().position(|&b| b == b'{') else { + return; + }; + let Ok(message) = serde_json::from_slice::(&payload[start..]) else { + return; + }; + if message.get("type").and_then(|t| t.as_str()) != Some("dsh") { + return; + } + let history: Vec = message + .get("history") + .and_then(|h| h.as_array()) + .map(|h| h.iter().filter_map(serde_json::Value::as_i64).collect()) + .unwrap_or_default(); + self.spoke(&history, "RTCP", session, tell); + } + /// Asks the meeting for the cameras to show, where that changed: one /// camera on of those in the call per camera line, each kept on its /// line while it stays on; none on a line once no one is left for it. diff --git a/src/teams/calling/media.rs b/src/teams/calling/media.rs index 04b235fe..a2a8258c 100644 --- a/src/teams/calling/media.rs +++ b/src/teams/calling/media.rs @@ -471,6 +471,14 @@ pub enum MediaEvent { ChannelOpen, /// A message on the meeting's data channel. ChannelData(Vec), + /// An application message in RTCP from the far end (`x-message app`: + /// the audio line offers `dsh` this way), with the sender's and the + /// media's SSRCs. + Feedback { + sender: u32, + media: u32, + payload: Vec, + }, /// The media broke; nothing more comes. Failed(Failure), /// Stopped as asked. @@ -1706,6 +1714,13 @@ impl Session { RtcEvent::ChannelClose(id) if Some(id) == self.channel => { log::info!("media: the meeting's data channel closed"); } + RtcEvent::AppSpecificFeedback(feedback) => { + let _ = self.tell.send(MediaEvent::Feedback { + sender: *feedback.sender_ssrc, + media: *feedback.media_ssrc, + payload: feedback.payload.to_vec(), + }); + } _ => {} } } diff --git a/src/teams/calling/types.rs b/src/teams/calling/types.rs index b917bda8..140d578b 100644 --- a/src/teams/calling/types.rs +++ b/src/teams/calling/types.rs @@ -1057,6 +1057,9 @@ pub enum Push { /// A renegotiation of ours was refused (`rejection`, /// `mediaNegotiationFailure`), by its event name. Refused(String), + /// The media controller says which contributing sources (CSRCs) are + /// whose; not yet understood, kept whole to be logged. + CsrcInfo(serde_json::Value), /// A callback a call does not act on (`progress`, /// `admitParticipantSuccess`, …), by its event name. Other(String), @@ -1087,6 +1090,7 @@ impl Push { "end" => Self::CallEnd(CallEndPush::deserialize(body)?.call_end), "conversationEnd" => Self::ConversationEnd(ConversationEnd::deserialize(body)?), "rejection" | "mediaNegotiationFailure" => Self::Refused(event.to_owned()), + "csrcInfo" => Self::CsrcInfo(body.clone()), other => Self::Other(other.to_owned()), }) }