From b29af698e3cf45da4e077ffff1d9d692fdf055d0 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 01:32:19 +0200 Subject: [PATCH 1/4] fix: play and record on Windows devices that mix in more than two channels WebRTC's Windows audio device module asks a device only for mono or stereo. A shared-mode stream must have the channel count of the device's mix format, so a device that mixes in 7.1, as some gaming headsets do, rejects every format, and initializing playout fails with no sound. A WebRTC patch (patches/windows/wasapi_mix_format.patch) handles that case the way Chrome and Firefox do: when no mono or stereo format is accepted, the stream is opened in the device's mix format (16-bit PCM in its channel count, speaker mask and rate), and the module converts between it and WebRTC's mono or stereo at the WASAPI buffer, with WebRTC's port of Chrome's channel mixing matrix. WebRTC above the module keeps getting one or two channels, which is all it accepts. Devices that accept mono or stereo are opened exactly as before. The WebRTC cache key on CI now covers the patches, so a changed patch rebuilds WebRTC instead of restoring a build without it. PlayoutIntegrationTest plays out on the default speaker and checks that WebRTC is asked for mono or stereo 10 ms blocks; it is skipped on hosts without a playout device. --- .github/actions/build/action.yml | 4 +- .../patches/windows/wasapi_mix_format.patch | 339 ++++++++++++++++++ .../media/audio/PlayoutIntegrationTest.java | 110 ++++++ 3 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/media/audio/PlayoutIntegrationTest.java diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 83692d53d..c7250475c 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -38,8 +38,8 @@ runs: uses: actions/cache/restore@v6 with: path: ~/${{ env.WEBRTC_INSTALL_FOLDER }} - key: webrtc-${{ env.WEBRTC_CACHE_BRANCH }}-${{ inputs.platform-name }}-${{ hashFiles('webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt') }}-${{ hashFiles('webrtc-jni/pom.xml') }} - restore-keys: webrtc-${{ env.WEBRTC_CACHE_BRANCH }}-${{ inputs.platform-name }}-${{ hashFiles('webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt') }}- + key: webrtc-${{ env.WEBRTC_CACHE_BRANCH }}-${{ inputs.platform-name }}-${{ hashFiles('webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt', 'webrtc-jni/src/main/cpp/dependencies/webrtc/patches/**/*.patch') }}-${{ hashFiles('webrtc-jni/pom.xml') }} + restore-keys: webrtc-${{ env.WEBRTC_CACHE_BRANCH }}-${{ inputs.platform-name }}-${{ hashFiles('webrtc-jni/src/main/cpp/dependencies/webrtc/CMakeLists.txt', 'webrtc-jni/src/main/cpp/dependencies/webrtc/patches/**/*.patch') }}- - name: Derive the FFmpeg version from the pom run: echo "FFMPEG_VERSION=$(sed -n 's|.*\([^<]*\).*|\1|p' webrtc-java-media/pom.xml)" >> "$GITHUB_ENV" diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch new file mode 100644 index 000000000..d6f5331af --- /dev/null +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch @@ -0,0 +1,339 @@ +--- a/modules/audio_device/BUILD.gn ++++ b/modules/audio_device/BUILD.gn +@@ -261,9 +261,11 @@ + "../../api:scoped_refptr", + "../../api:sequence_checker", + "../../api/audio:audio_device", ++ "../../api/audio:audio_frame_api", + "../../api/environment", + "../../api/task_queue", + "../../api/units:time_delta", ++ "../../audio/utility:audio_frame_operations", + "../../common_audio", + "../../common_audio:common_audio_c", + "../../rtc_base:buffer", +--- a/modules/audio_device/win/audio_device_core_win.cc ++++ b/modules/audio_device/win/audio_device_core_win.cc +@@ -25,9 +25,13 @@ + #include + #include + ++#include ++#include + #include + ++#include "api/audio/channel_layout.h" + #include "api/make_ref_counted.h" ++#include "audio/utility/channel_mixing_matrix.h" + #include "rtc_base/checks.h" + #include "rtc_base/logging.h" + #include "rtc_base/platform_thread.h" +@@ -77,6 +81,129 @@ + + enum { kAecCaptureStreamIndex = 0, kAecRenderStreamIndex = 1 }; + ++// Returns the channel layout of a WASAPI format, mapping the speaker masks ++// the way Chrome's ChannelConfigToChannelLayout() does. A mask that is not ++// one of these, or that does not match the channel count, is treated as ++// discrete channels, which are copied by index. ++ChannelLayout WaveFormatToChannelLayout(const WAVEFORMATEX* format) { ++ ChannelLayout layout = GuessChannelLayout(format->nChannels); ++ if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && format->cbSize >= 22) { ++ switch ( ++ reinterpret_cast(format)->dwChannelMask) { ++ case KSAUDIO_SPEAKER_MONO: ++ layout = CHANNEL_LAYOUT_MONO; ++ break; ++ case KSAUDIO_SPEAKER_STEREO: ++ layout = CHANNEL_LAYOUT_STEREO; ++ break; ++ case KSAUDIO_SPEAKER_QUAD: ++ layout = CHANNEL_LAYOUT_QUAD; ++ break; ++ case KSAUDIO_SPEAKER_SURROUND: ++ layout = CHANNEL_LAYOUT_4_0; ++ break; ++ case KSAUDIO_SPEAKER_5POINT1: ++ layout = CHANNEL_LAYOUT_5_1_BACK; ++ break; ++ case KSAUDIO_SPEAKER_5POINT1_SURROUND: ++ layout = CHANNEL_LAYOUT_5_1; ++ break; ++ case KSAUDIO_SPEAKER_7POINT1: ++ layout = CHANNEL_LAYOUT_7_1_WIDE; ++ break; ++ case KSAUDIO_SPEAKER_7POINT1_SURROUND: ++ layout = CHANNEL_LAYOUT_7_1; ++ break; ++ default: ++ break; ++ } ++ } ++ if (layout == CHANNEL_LAYOUT_UNSUPPORTED || ++ ChannelLayoutToChannelCount(layout) != format->nChannels) { ++ layout = CHANNEL_LAYOUT_DISCRETE; ++ } ++ return layout; ++} ++ ++// Builds the stream format for a device that accepts none of the formats ++// WebRTC asks for: 16-bit PCM in the channel count, speaker mask and sample ++// rate of the device's shared-mode mix format. The audio engine converts ++// 16-bit PCM to its float mix format itself, but not the channel count or ++// the rate, so these must match the mix format. Chrome and Firefox open such ++// devices the same way and convert the channels themselves. `matrix` gets the ++// conversion from `client_channels` to the device (render) or back ++// (capture), and stays empty when the channel counts match. Returns false if ++// the mix format cannot be used. ++bool MixFormatForClient(const WAVEFORMATEX* mix_format, ++ uint16_t client_channels, ++ bool render, ++ WAVEFORMATEXTENSIBLE* format, ++ std::vector>* matrix) { ++ matrix->clear(); ++ // WebRTC exchanges audio in 10 ms blocks of whole frames. ++ if (mix_format->nChannels == 0 || mix_format->nSamplesPerSec % 100 != 0) { ++ return false; ++ } ++ ++ *format = WAVEFORMATEXTENSIBLE(); ++ format->Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE; ++ format->Format.nChannels = mix_format->nChannels; ++ format->Format.nSamplesPerSec = mix_format->nSamplesPerSec; ++ format->Format.wBitsPerSample = 16; ++ format->Format.nBlockAlign = ++ format->Format.nChannels * format->Format.wBitsPerSample / 8; ++ format->Format.nAvgBytesPerSec = ++ format->Format.nSamplesPerSec * format->Format.nBlockAlign; ++ format->Format.cbSize = sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX); ++ format->Samples.wValidBitsPerSample = format->Format.wBitsPerSample; ++ format->SubFormat = KSDATAFORMAT_SUBTYPE_PCM; ++ if (mix_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && ++ mix_format->cbSize >= 22) { ++ format->dwChannelMask = ++ reinterpret_cast(mix_format) ++ ->dwChannelMask; ++ } ++ ++ if (format->Format.nChannels == client_channels) { ++ return true; ++ } ++ ++ const ChannelLayout client_layout = GuessChannelLayout(client_channels); ++ const ChannelLayout device_layout = ++ WaveFormatToChannelLayout(&format->Format); ++ if (render) { ++ ChannelMixingMatrix(client_layout, client_channels, device_layout, ++ format->Format.nChannels) ++ .CreateTransformationMatrix(matrix); ++ } else { ++ ChannelMixingMatrix(device_layout, format->Format.nChannels, client_layout, ++ client_channels) ++ .CreateTransformationMatrix(matrix); ++ } ++ return true; ++} ++ ++// Converts `frames` interleaved frames with `matrix`, which has one row per ++// output channel and one column per input channel. ++void MixChannels(const std::vector>& matrix, ++ const int16_t* input, ++ int16_t* output, ++ size_t frames) { ++ const size_t output_channels = matrix.size(); ++ const size_t input_channels = matrix[0].size(); ++ for (size_t frame = 0; frame < frames; ++frame) { ++ for (size_t out = 0; out < output_channels; ++out) { ++ float sample = 0.f; ++ for (size_t in = 0; in < input_channels; ++in) { ++ sample += matrix[out][in] * input[in]; ++ } ++ *output++ = static_cast( ++ std::clamp(std::lrintf(sample), -32768l, 32767l)); ++ } ++ input += input_channels; ++ } ++} ++ + // An implementation of IMediaBuffer, as required for + // IMediaObject::ProcessOutput(). After consuming data provided by + // ProcessOutput(), call SetLength() to update the buffer availability. +@@ -1822,6 +1949,8 @@ + HRESULT hr = S_OK; + WAVEFORMATEX* pWfxOut = NULL; + WAVEFORMATEX Wfx = WAVEFORMATEX(); ++ WAVEFORMATEXTENSIBLE WfxMix = WAVEFORMATEXTENSIBLE(); ++ WAVEFORMATEX* pWfxSelected = &Wfx; + WAVEFORMATEX* pWfxClosestMatch = NULL; + UINT bufferFrameCount = 0; + constexpr int freqs[] = {48000, 44100, 16000, 96000, 32000, 8000}; +@@ -1901,6 +2030,7 @@ + // TODO(andrew): what happens in the event of failure in the above loop? + // Is _ptrClientOut->Initialize expected to fail? + // Same in InitRecording(). ++ _playMixMatrix.clear(); + if (hr == S_OK) { + _playAudioFrameSize = Wfx.nBlockAlign; + // Block size is the number of samples each channel in 10ms. +@@ -1924,6 +2054,29 @@ + RTC_LOG(LS_VERBOSE) << "_playAudioFrameSize: " << _playAudioFrameSize; + RTC_LOG(LS_VERBOSE) << "_playBlockSize : " << _playBlockSize; + RTC_LOG(LS_VERBOSE) << "_playChannels : " << _playChannels; ++ } else if (pWfxOut && ++ MixFormatForClient(pWfxOut, _playChannelsPrioList[0], ++ /*render=*/true, &WfxMix, &_playMixMatrix)) { ++ // None of the formats above is supported. This happens when the mix ++ // format has more than two channels, as with gaming headsets that mix in ++ // 7.1, since a shared-mode stream must have the channel count of the mix ++ // format. Open the stream in the mix format and convert WebRTC's audio to ++ // it in the render thread. ++ pWfxSelected = &WfxMix.Format; ++ _playAudioFrameSize = WfxMix.Format.nBlockAlign; ++ _playBlockSize = WfxMix.Format.nSamplesPerSec / 100; ++ _playSampleRate = WfxMix.Format.nSamplesPerSec; ++ _devicePlaySampleRate = WfxMix.Format.nSamplesPerSec; ++ _devicePlayBlockSize = WfxMix.Format.nSamplesPerSec / 100; ++ _playChannels = _playChannelsPrioList[0]; ++ _playConvertBuffer.resize(_playBlockSize * _playChannels); ++ ++ RTC_LOG(LS_INFO) << "Rendering in the mix format: nChannels=" ++ << WfxMix.Format.nChannels ++ << ", nSamplesPerSec=" << WfxMix.Format.nSamplesPerSec ++ << ", dwChannelMask=0x" ++ << ToHex(static_cast(WfxMix.dwChannelMask)) ++ << ", converted from " << _playChannels << " channel(s)"; + } + + // Create a rendering stream. +@@ -1959,7 +2112,7 @@ + hnsBufferDuration, // requested buffer capacity as a time value (in + // 100-nanosecond units) + 0, // periodicity +- &Wfx, // selected wave format ++ pWfxSelected, // selected wave format + NULL); // session GUID + + if (FAILED(hr)) { +@@ -2056,6 +2209,7 @@ + ptrWav->cbSize = 0; + + // Set the VoE format equal to the AEC output format. ++ _recMixMatrix.clear(); + _recAudioFrameSize = ptrWav->nBlockAlign; + _recSampleRate = ptrWav->nSamplesPerSec; + _recBlockSize = ptrWav->nSamplesPerSec / 100; +@@ -2136,6 +2290,8 @@ + HRESULT hr = S_OK; + WAVEFORMATEX* pWfxIn = NULL; + WAVEFORMATEXTENSIBLE Wfx = WAVEFORMATEXTENSIBLE(); ++ WAVEFORMATEXTENSIBLE WfxMix = WAVEFORMATEXTENSIBLE(); ++ WAVEFORMATEXTENSIBLE* pWfxSelected = &Wfx; + WAVEFORMATEX* pWfxClosestMatch = NULL; + UINT bufferFrameCount = 0; + constexpr int freqs[6] = {48000, 44100, 16000, 96000, 32000, 8000}; +@@ -2216,6 +2372,7 @@ + break; + } + ++ _recMixMatrix.clear(); + if (hr == S_OK) { + _recAudioFrameSize = Wfx.Format.nBlockAlign; + _recSampleRate = Wfx.Format.nSamplesPerSec; +@@ -2236,6 +2393,25 @@ + RTC_LOG(LS_VERBOSE) << "_recAudioFrameSize: " << _recAudioFrameSize; + RTC_LOG(LS_VERBOSE) << "_recBlockSize : " << _recBlockSize; + RTC_LOG(LS_VERBOSE) << "_recChannels : " << _recChannels; ++ } else if (pWfxIn && ++ MixFormatForClient(pWfxIn, _recChannelsPrioList[0], ++ /*render=*/false, &WfxMix, &_recMixMatrix)) { ++ // None of the formats above is supported; see InitPlayout(). Open the ++ // stream in the mix format and convert it to WebRTC's channel count in ++ // the capture thread. `_recAudioFrameSize` stays the size of a frame in ++ // WebRTC's format, which is what the capture thread buffers. ++ pWfxSelected = &WfxMix; ++ _recChannels = _recChannelsPrioList[0]; ++ _recAudioFrameSize = _recChannels * WfxMix.Format.wBitsPerSample / 8; ++ _recSampleRate = WfxMix.Format.nSamplesPerSec; ++ _recBlockSize = WfxMix.Format.nSamplesPerSec / 100; ++ ++ RTC_LOG(LS_INFO) << "Capturing in the mix format: nChannels=" ++ << WfxMix.Format.nChannels ++ << ", nSamplesPerSec=" << WfxMix.Format.nSamplesPerSec ++ << ", dwChannelMask=0x" ++ << ToHex(static_cast(WfxMix.dwChannelMask)) ++ << ", converted to " << _recChannels << " channel(s)"; + } + + // Create a capturing stream. +@@ -2246,10 +2422,10 @@ + AUDCLNT_STREAMFLAGS_NOPERSIST, // volume and mute settings for an + // audio session will not persist + // across system restarts +- 0, // required for event-driven shared mode +- 0, // periodicity +- (WAVEFORMATEX*)&Wfx, // selected wave format +- NULL); // session GUID ++ 0, // required for event-driven shared mode ++ 0, // periodicity ++ (WAVEFORMATEX*)pWfxSelected, // selected wave format ++ NULL); // session GUID + + if (hr != S_OK) { + RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; +@@ -2832,7 +3008,14 @@ + } + + // Get the actual (stored) data +- nSamples = _ptrAudioBuffer->GetPlayoutData((int8_t*)pData); ++ if (_playMixMatrix.empty()) { ++ nSamples = _ptrAudioBuffer->GetPlayoutData((int8_t*)pData); ++ } else { ++ nSamples = ++ _ptrAudioBuffer->GetPlayoutData(_playConvertBuffer.data()); ++ MixChannels(_playMixMatrix, _playConvertBuffer.data(), ++ reinterpret_cast(pData), _playBlockSize); ++ } + } + + DWORD dwFlags(0); +@@ -3221,7 +3404,12 @@ + + RTC_DCHECK_NE(framesAvailable, 0); + +- if (pData) { ++ if (pData && !_recMixMatrix.empty()) { ++ MixChannels(_recMixMatrix, reinterpret_cast(pData), ++ reinterpret_cast( ++ &syncBuffer[syncBufIndex * _recAudioFrameSize]), ++ framesAvailable); ++ } else if (pData) { + CopyMemory(&syncBuffer[syncBufIndex * _recAudioFrameSize], pData, + framesAvailable * _recAudioFrameSize); + } else { +--- a/modules/audio_device/win/audio_device_core_win.h ++++ b/modules/audio_device/win/audio_device_core_win.h +@@ -21,6 +21,8 @@ + #include // IMediaObject + #include // MMDevice + ++#include ++ + #include "api/scoped_refptr.h" + #include "modules/audio_device/audio_device_generic.h" + #include "rtc_base/synchronization/mutex.h" +@@ -270,6 +272,15 @@ + uint16_t _recChannelsPrioList[3]; + uint16_t _playChannelsPrioList[2]; + ++ // Channel conversion for a device that is opened in its own mix format, ++ // because it accepts none of the formats above. Each matrix maps WebRTC's ++ // mono or stereo to the device's channels (render), or back (capture), and ++ // is empty when no conversion is needed. `_playConvertBuffer` holds one ++ // 10 ms block in WebRTC's channel count before it is converted. ++ std::vector> _playMixMatrix; ++ std::vector> _recMixMatrix; ++ std::vector _playConvertBuffer; ++ + LARGE_INTEGER _perfCounterFreq; + double _perfCounterFactor; + diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/PlayoutIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/PlayoutIntegrationTest.java new file mode 100644 index 000000000..b13220d18 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/PlayoutIntegrationTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.audio; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import dev.onvoid.webrtc.media.MediaDevices; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Plays out on the default speaker through the platform audio device module. + * Needs a real playout device and is skipped without one. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) +class PlayoutIntegrationTest { + + @Test + @EnabledOnOs(OS.WINDOWS) + void playoutOnDefaultSpeaker() throws Exception { + AudioDevice speaker = null; + + try { + speaker = MediaDevices.getDefaultAudioRenderDevice(); + } + catch (RuntimeException e) { + // No audio system on this host. + } + + assumeTrue(speaker != null, "no playout device"); + + // WebRTC exchanges mono or stereo 10 ms blocks with the module, whatever + // the device mixes in. A device whose shared-mode mix format has more + // channels, as gaming headsets that mix in 7.1 have, accepts no stereo + // stream, so the module opens it in its mix format and converts. Without + // that, initPlayout() fails on such a device and no block arrives. + CountDownLatch blocks = new CountDownLatch(20); + AtomicReference wrongFormat = new AtomicReference<>(); + + AudioDeviceModule module = new AudioDeviceModule(); + + try { + module.setPlayoutDevice(speaker); + module.setAudioSource(new AudioSource() { + + @Override + public int onPlaybackData(byte[] audioSamples, int nSamples, + int nBytesPerSample, int nChannels, int samplesPerSec) { + // This runs on the module's render thread, where a thrown + // assertion is lost and even printing fails under Surefire, + // so the format is recorded and checked on the test thread. + if (nChannels < 1 || nChannels > 2 + || nBytesPerSample != 2 * nChannels + || nSamples * 100 != samplesPerSec) { + wrongFormat.compareAndSet(null, String.format( + "%d frames, %d bytes per frame, %d channels, %d Hz", + nSamples, nBytesPerSample, nChannels, + samplesPerSec)); + } + + blocks.countDown(); + + // The buffer is silent; play it. + return nSamples; + } + }); + + module.initPlayout(); + module.startPlayout(); + + try { + assertTrue(blocks.await(5, TimeUnit.SECONDS), + "playout did not run"); + assertNull(wrongFormat.get(), "unexpected playout format"); + } + finally { + module.stopPlayout(); + } + } + finally { + module.dispose(); + } + } +} From 04bf94f8d18803be531af3240c5a6afc35448fda Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 16:54:07 +0200 Subject: [PATCH 2/4] perf: copy channels instead of mixing them when the matrix only routes them When a device is opened in its mix format, the module converted every sample through the full channel matrix. For playout to more channels than WebRTC's, which is the case of a 7.1 headset, that matrix only routes channels: left and right go to front left and right, the rest is silent, and mono goes to both. ChannelMixingMatrix reports this, and Chrome's ChannelMixer then copies instead of mixing. The module does the same now, with a map of the input channel of each output channel, since its samples are interleaved. The output is the same. On 10 ms at 48 kHz, stereo to 7.1 drops from about 10 to about 1 us per block. Capture from 7.1 to stereo mixes, and keeps the matrix. --- .../patches/windows/wasapi_mix_format.patch | 118 +++++++++++++----- 1 file changed, 84 insertions(+), 34 deletions(-) diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch index d6f5331af..2989662eb 100644 --- a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch @@ -28,7 +28,7 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/platform_thread.h" -@@ -77,6 +81,129 @@ +@@ -77,6 +81,159 @@ enum { kAecCaptureStreamIndex = 0, kAecRenderStreamIndex = 1 }; @@ -83,14 +83,18 @@ +// the rate, so these must match the mix format. Chrome and Firefox open such +// devices the same way and convert the channels themselves. `matrix` gets the +// conversion from `client_channels` to the device (render) or back -+// (capture), and stays empty when the channel counts match. Returns false if -+// the mix format cannot be used. ++// (capture), and stays empty when the channel counts match. When the matrix ++// only routes channels, `channel_map` gets the input channel of each output ++// channel, or -1 for silence, and stays empty otherwise. Returns false if the ++// mix format cannot be used. +bool MixFormatForClient(const WAVEFORMATEX* mix_format, + uint16_t client_channels, + bool render, + WAVEFORMATEXTENSIBLE* format, -+ std::vector>* matrix) { ++ std::vector>* matrix, ++ std::vector* channel_map) { + matrix->clear(); ++ channel_map->clear(); + // WebRTC exchanges audio in 10 ms blocks of whole frames. + if (mix_format->nChannels == 0 || mix_format->nSamplesPerSec % 100 != 0) { + return false; @@ -122,18 +126,44 @@ + const ChannelLayout client_layout = GuessChannelLayout(client_channels); + const ChannelLayout device_layout = + WaveFormatToChannelLayout(&format->Format); -+ if (render) { -+ ChannelMixingMatrix(client_layout, client_channels, device_layout, -+ format->Format.nChannels) -+ .CreateTransformationMatrix(matrix); -+ } else { -+ ChannelMixingMatrix(device_layout, format->Format.nChannels, client_layout, -+ client_channels) -+ .CreateTransformationMatrix(matrix); ++ const bool remapping = ++ render ? ChannelMixingMatrix(client_layout, client_channels, ++ device_layout, format->Format.nChannels) ++ .CreateTransformationMatrix(matrix) ++ : ChannelMixingMatrix(device_layout, format->Format.nChannels, ++ client_layout, client_channels) ++ .CreateTransformationMatrix(matrix); ++ if (remapping) { ++ // Each output channel takes one input channel as it is, or none. ++ channel_map->assign(matrix->size(), -1); ++ for (size_t out = 0; out < matrix->size(); ++out) { ++ for (size_t in = 0; in < (*matrix)[out].size(); ++in) { ++ if ((*matrix)[out][in] > 0) { ++ (*channel_map)[out] = static_cast(in); ++ break; ++ } ++ } ++ } + } + return true; +} + ++// Copies `frames` interleaved frames by channel: `channel_map` names the input ++// channel of each output channel, or -1 for silence. ++void RemapChannels(const std::vector& channel_map, ++ size_t input_channels, ++ const int16_t* input, ++ int16_t* output, ++ size_t frames) { ++ const size_t output_channels = channel_map.size(); ++ for (size_t frame = 0; frame < frames; ++frame) { ++ for (size_t out = 0; out < output_channels; ++out) { ++ *output++ = channel_map[out] < 0 ? 0 : input[channel_map[out]]; ++ } ++ input += input_channels; ++ } ++} ++ +// Converts `frames` interleaved frames with `matrix`, which has one row per +// output channel and one column per input channel. +void MixChannels(const std::vector>& matrix, @@ -158,7 +188,7 @@ // An implementation of IMediaBuffer, as required for // IMediaObject::ProcessOutput(). After consuming data provided by // ProcessOutput(), call SetLength() to update the buffer availability. -@@ -1822,6 +1949,8 @@ +@@ -1822,6 +1979,8 @@ HRESULT hr = S_OK; WAVEFORMATEX* pWfxOut = NULL; WAVEFORMATEX Wfx = WAVEFORMATEX(); @@ -167,21 +197,22 @@ WAVEFORMATEX* pWfxClosestMatch = NULL; UINT bufferFrameCount = 0; constexpr int freqs[] = {48000, 44100, 16000, 96000, 32000, 8000}; -@@ -1901,6 +2030,7 @@ +@@ -1901,6 +2060,8 @@ // TODO(andrew): what happens in the event of failure in the above loop? // Is _ptrClientOut->Initialize expected to fail? // Same in InitRecording(). + _playMixMatrix.clear(); ++ _playChannelMap.clear(); if (hr == S_OK) { _playAudioFrameSize = Wfx.nBlockAlign; // Block size is the number of samples each channel in 10ms. -@@ -1924,6 +2054,29 @@ +@@ -1924,6 +2085,29 @@ RTC_LOG(LS_VERBOSE) << "_playAudioFrameSize: " << _playAudioFrameSize; RTC_LOG(LS_VERBOSE) << "_playBlockSize : " << _playBlockSize; RTC_LOG(LS_VERBOSE) << "_playChannels : " << _playChannels; -+ } else if (pWfxOut && -+ MixFormatForClient(pWfxOut, _playChannelsPrioList[0], -+ /*render=*/true, &WfxMix, &_playMixMatrix)) { ++ } else if (pWfxOut && MixFormatForClient(pWfxOut, _playChannelsPrioList[0], ++ /*render=*/true, &WfxMix, ++ &_playMixMatrix, &_playChannelMap)) { + // None of the formats above is supported. This happens when the mix + // format has more than two channels, as with gaming headsets that mix in + // 7.1, since a shared-mode stream must have the channel count of the mix @@ -205,7 +236,7 @@ } // Create a rendering stream. -@@ -1959,7 +2112,7 @@ +@@ -1959,7 +2143,7 @@ hnsBufferDuration, // requested buffer capacity as a time value (in // 100-nanosecond units) 0, // periodicity @@ -214,15 +245,16 @@ NULL); // session GUID if (FAILED(hr)) { -@@ -2056,6 +2209,7 @@ +@@ -2056,6 +2240,8 @@ ptrWav->cbSize = 0; // Set the VoE format equal to the AEC output format. + _recMixMatrix.clear(); ++ _recChannelMap.clear(); _recAudioFrameSize = ptrWav->nBlockAlign; _recSampleRate = ptrWav->nSamplesPerSec; _recBlockSize = ptrWav->nSamplesPerSec / 100; -@@ -2136,6 +2290,8 @@ +@@ -2136,6 +2322,8 @@ HRESULT hr = S_OK; WAVEFORMATEX* pWfxIn = NULL; WAVEFORMATEXTENSIBLE Wfx = WAVEFORMATEXTENSIBLE(); @@ -231,21 +263,22 @@ WAVEFORMATEX* pWfxClosestMatch = NULL; UINT bufferFrameCount = 0; constexpr int freqs[6] = {48000, 44100, 16000, 96000, 32000, 8000}; -@@ -2216,6 +2372,7 @@ +@@ -2216,6 +2404,8 @@ break; } + _recMixMatrix.clear(); ++ _recChannelMap.clear(); if (hr == S_OK) { _recAudioFrameSize = Wfx.Format.nBlockAlign; _recSampleRate = Wfx.Format.nSamplesPerSec; -@@ -2236,6 +2393,25 @@ +@@ -2236,6 +2426,25 @@ RTC_LOG(LS_VERBOSE) << "_recAudioFrameSize: " << _recAudioFrameSize; RTC_LOG(LS_VERBOSE) << "_recBlockSize : " << _recBlockSize; RTC_LOG(LS_VERBOSE) << "_recChannels : " << _recChannels; -+ } else if (pWfxIn && -+ MixFormatForClient(pWfxIn, _recChannelsPrioList[0], -+ /*render=*/false, &WfxMix, &_recMixMatrix)) { ++ } else if (pWfxIn && MixFormatForClient(pWfxIn, _recChannelsPrioList[0], ++ /*render=*/false, &WfxMix, ++ &_recMixMatrix, &_recChannelMap)) { + // None of the formats above is supported; see InitPlayout(). Open the + // stream in the mix format and convert it to WebRTC's channel count in + // the capture thread. `_recAudioFrameSize` stays the size of a frame in @@ -265,7 +298,7 @@ } // Create a capturing stream. -@@ -2246,10 +2422,10 @@ +@@ -2246,10 +2455,10 @@ AUDCLNT_STREAMFLAGS_NOPERSIST, // volume and mute settings for an // audio session will not persist // across system restarts @@ -280,7 +313,7 @@ if (hr != S_OK) { RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; -@@ -2832,7 +3008,14 @@ +@@ -2832,7 +3041,20 @@ } // Get the actual (stored) data @@ -290,18 +323,30 @@ + } else { + nSamples = + _ptrAudioBuffer->GetPlayoutData(_playConvertBuffer.data()); -+ MixChannels(_playMixMatrix, _playConvertBuffer.data(), -+ reinterpret_cast(pData), _playBlockSize); ++ if (!_playChannelMap.empty()) { ++ RemapChannels(_playChannelMap, _playChannels, ++ _playConvertBuffer.data(), ++ reinterpret_cast(pData), _playBlockSize); ++ } else { ++ MixChannels(_playMixMatrix, _playConvertBuffer.data(), ++ reinterpret_cast(pData), _playBlockSize); ++ } + } } DWORD dwFlags(0); -@@ -3221,7 +3404,12 @@ +@@ -3221,7 +3443,18 @@ RTC_DCHECK_NE(framesAvailable, 0); - if (pData) { -+ if (pData && !_recMixMatrix.empty()) { ++ if (pData && !_recChannelMap.empty()) { ++ RemapChannels(_recChannelMap, _recMixMatrix[0].size(), ++ reinterpret_cast(pData), ++ reinterpret_cast( ++ &syncBuffer[syncBufIndex * _recAudioFrameSize]), ++ framesAvailable); ++ } else if (pData && !_recMixMatrix.empty()) { + MixChannels(_recMixMatrix, reinterpret_cast(pData), + reinterpret_cast( + &syncBuffer[syncBufIndex * _recAudioFrameSize]), @@ -321,17 +366,22 @@ #include "api/scoped_refptr.h" #include "modules/audio_device/audio_device_generic.h" #include "rtc_base/synchronization/mutex.h" -@@ -270,6 +272,15 @@ +@@ -270,6 +272,20 @@ uint16_t _recChannelsPrioList[3]; uint16_t _playChannelsPrioList[2]; + // Channel conversion for a device that is opened in its own mix format, + // because it accepts none of the formats above. Each matrix maps WebRTC's + // mono or stereo to the device's channels (render), or back (capture), and -+ // is empty when no conversion is needed. `_playConvertBuffer` holds one ++ // is empty when no conversion is needed. When a matrix only routes ++ // channels, the channel map names the input channel of each output channel ++ // (-1 for silence) and the samples are copied instead of mixed, as Chrome's ++ // ChannelMixer does; it is empty otherwise. `_playConvertBuffer` holds one + // 10 ms block in WebRTC's channel count before it is converted. + std::vector> _playMixMatrix; + std::vector> _recMixMatrix; ++ std::vector _playChannelMap; ++ std::vector _recChannelMap; + std::vector _playConvertBuffer; + LARGE_INTEGER _perfCounterFreq; From 1d92120ecc6fd550e90380998077ed6dae53a0fc Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 17:17:24 +0200 Subject: [PATCH 3/4] perf: mix channels the way Firefox's cubeb mixer does Channels that have to be mixed, rather than only routed, went through the full float matrix for every sample: every coefficient, zeros included, then a rounding call per output sample. They are now mixed the way Firefox's cubeb mixer mixes 16-bit audio: each output channel keeps only its non-zero terms, as Q15 integer coefficients, with loops for none, one, two and more terms, rounding by (sum + 16384) >> 15, and clamping only when a matrix can leave the 16-bit range. Unlike cubeb, sums are taken in 64 bits. A row of WebRTC's 7.1 or 5.1 downmix sums to 3.1 to 6 in coefficients, so full-scale input on every channel reaches 3.4 to 6.4 billion, past what cubeb's 32-bit int holds. The matrix and the channel map become one CoreAudioChannelConversion per direction, which ConvertChannels() applies: a copy when the matrix only routes channels, as before, the terms otherwise. On 10 ms at 48 kHz, capture from 7.1 to stereo drops from about 8.5 to 4.3 us, 5.1 to stereo from 7.3 to 3.8, 7.1 to mono from 3.8 to 3.0, and playout from stereo to mono from 2.9 to 0.4. The output is that of the float matrix within 1 to 2 of 32768, and the same at full scale. --- .../patches/windows/wasapi_mix_format.patch | 281 +++++++++++------- 1 file changed, 179 insertions(+), 102 deletions(-) diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch index 2989662eb..6fe92d3fe 100644 --- a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch @@ -14,12 +14,13 @@ "../../rtc_base:buffer", --- a/modules/audio_device/win/audio_device_core_win.cc +++ b/modules/audio_device/win/audio_device_core_win.cc -@@ -25,9 +25,13 @@ +@@ -25,9 +25,14 @@ #include #include +#include +#include ++#include #include +#include "api/audio/channel_layout.h" @@ -28,7 +29,7 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/platform_thread.h" -@@ -77,6 +81,159 @@ +@@ -77,6 +82,225 @@ enum { kAecCaptureStreamIndex = 0, kAecRenderStreamIndex = 1 }; @@ -81,20 +82,16 @@ +// rate of the device's shared-mode mix format. The audio engine converts +// 16-bit PCM to its float mix format itself, but not the channel count or +// the rate, so these must match the mix format. Chrome and Firefox open such -+// devices the same way and convert the channels themselves. `matrix` gets the -+// conversion from `client_channels` to the device (render) or back -+// (capture), and stays empty when the channel counts match. When the matrix -+// only routes channels, `channel_map` gets the input channel of each output -+// channel, or -1 for silence, and stays empty otherwise. Returns false if the -+// mix format cannot be used. ++// devices the same way and convert the channels themselves. `conversion` gets ++// the conversion from `client_channels` to the device (render) or back ++// (capture), and stays empty when the channel counts match. Returns false if ++// the mix format cannot be used. +bool MixFormatForClient(const WAVEFORMATEX* mix_format, + uint16_t client_channels, + bool render, + WAVEFORMATEXTENSIBLE* format, -+ std::vector>* matrix, -+ std::vector* channel_map) { -+ matrix->clear(); -+ channel_map->clear(); ++ CoreAudioChannelConversion* conversion) { ++ *conversion = CoreAudioChannelConversion(); + // WebRTC exchanges audio in 10 ms blocks of whole frames. + if (mix_format->nChannels == 0 || mix_format->nSamplesPerSec % 100 != 0) { + return false; @@ -126,69 +123,139 @@ + const ChannelLayout client_layout = GuessChannelLayout(client_channels); + const ChannelLayout device_layout = + WaveFormatToChannelLayout(&format->Format); ++ std::vector> matrix; + const bool remapping = + render ? ChannelMixingMatrix(client_layout, client_channels, + device_layout, format->Format.nChannels) -+ .CreateTransformationMatrix(matrix) ++ .CreateTransformationMatrix(&matrix) + : ChannelMixingMatrix(device_layout, format->Format.nChannels, + client_layout, client_channels) -+ .CreateTransformationMatrix(matrix); ++ .CreateTransformationMatrix(&matrix); ++ conversion->input_channels = matrix[0].size(); + if (remapping) { + // Each output channel takes one input channel as it is, or none. -+ channel_map->assign(matrix->size(), -1); -+ for (size_t out = 0; out < matrix->size(); ++out) { -+ for (size_t in = 0; in < (*matrix)[out].size(); ++in) { -+ if ((*matrix)[out][in] > 0) { -+ (*channel_map)[out] = static_cast(in); ++ conversion->channel_map.assign(matrix.size(), -1); ++ for (size_t out = 0; out < matrix.size(); ++out) { ++ for (size_t in = 0; in < matrix[out].size(); ++in) { ++ if (matrix[out][in] > 0) { ++ conversion->channel_map[out] = static_cast(in); + break; + } + } + } ++ return true; ++ } ++ conversion->terms.resize(matrix.size()); ++ for (size_t out = 0; out < matrix.size(); ++out) { ++ int64_t sum = 0; ++ for (size_t in = 0; in < matrix[out].size(); ++in) { ++ const int32_t coefficient = ++ static_cast(std::lrintf(matrix[out][in] * 32768)); ++ if (coefficient != 0) { ++ conversion->terms[out].emplace_back(in, coefficient); ++ sum += std::abs(coefficient); ++ } ++ } ++ // Full-scale input on every term would leave the 16-bit range. ++ if (sum > 32768) { ++ conversion->clipping = true; ++ } + } + return true; +} + -+// Copies `frames` interleaved frames by channel: `channel_map` names the input -+// channel of each output channel, or -1 for silence. -+void RemapChannels(const std::vector& channel_map, -+ size_t input_channels, -+ const int16_t* input, -+ int16_t* output, -+ size_t frames) { -+ const size_t output_channels = channel_map.size(); -+ for (size_t frame = 0; frame < frames; ++frame) { -+ for (size_t out = 0; out < output_channels; ++out) { -+ *output++ = channel_map[out] < 0 ? 0 : input[channel_map[out]]; ++// Rounds a sum of Q15 terms to a 16-bit sample, as Firefox's cubeb mixer ++// does, and clamps it when `kClamp`. ++template ++int16_t RoundQ15(int64_t sum) { ++ int64_t sample = (sum + 16384) >> 15; ++ if (kClamp) { ++ sample = std::clamp(sample, -32768, 32767); ++ } ++ return static_cast(sample); ++} ++ ++// Mixes `frames` interleaved frames with the terms of `conversion`, one output ++// channel at a time and with loops for the usual numbers of terms, as ++// Firefox's cubeb mixer does. Sums are taken in 64 bits: cubeb's 32-bit sums ++// can overflow when several full-scale inputs mix into one channel, as in a ++// downmix of 7.1. ++template ++void MixTerms(const CoreAudioChannelConversion& conversion, ++ const int16_t* input, ++ int16_t* output, ++ size_t frames) { ++ const size_t input_channels = conversion.input_channels; ++ const size_t output_channels = conversion.terms.size(); ++ for (size_t out = 0; out < output_channels; ++out) { ++ const auto& terms = conversion.terms[out]; ++ int16_t* dest = output + out; ++ switch (terms.size()) { ++ case 0: ++ for (size_t frame = 0; frame < frames; ++frame) { ++ dest[frame * output_channels] = 0; ++ } ++ break; ++ case 1: { ++ const int16_t* a = input + terms[0].first; ++ const int64_t ca = terms[0].second; ++ for (size_t frame = 0; frame < frames; ++frame) { ++ dest[frame * output_channels] = ++ RoundQ15(ca * a[frame * input_channels]); ++ } ++ break; ++ } ++ case 2: { ++ const int16_t* a = input + terms[0].first; ++ const int16_t* b = input + terms[1].first; ++ const int64_t ca = terms[0].second; ++ const int64_t cb = terms[1].second; ++ for (size_t frame = 0; frame < frames; ++frame) { ++ dest[frame * output_channels] = RoundQ15( ++ ca * a[frame * input_channels] + cb * b[frame * input_channels]); ++ } ++ break; ++ } ++ default: ++ for (size_t frame = 0; frame < frames; ++frame) { ++ const int16_t* in = input + frame * input_channels; ++ int64_t sum = 0; ++ for (const auto& [channel, coefficient] : terms) { ++ sum += int64_t{coefficient} * in[channel]; ++ } ++ dest[frame * output_channels] = RoundQ15(sum); ++ } ++ break; + } -+ input += input_channels; + } +} + -+// Converts `frames` interleaved frames with `matrix`, which has one row per -+// output channel and one column per input channel. -+void MixChannels(const std::vector>& matrix, -+ const int16_t* input, -+ int16_t* output, -+ size_t frames) { -+ const size_t output_channels = matrix.size(); -+ const size_t input_channels = matrix[0].size(); -+ for (size_t frame = 0; frame < frames; ++frame) { -+ for (size_t out = 0; out < output_channels; ++out) { -+ float sample = 0.f; -+ for (size_t in = 0; in < input_channels; ++in) { -+ sample += matrix[out][in] * input[in]; ++// Converts `frames` interleaved frames with `conversion`: copies them by ++// channel when it only routes channels, and mixes them otherwise. ++void ConvertChannels(const CoreAudioChannelConversion& conversion, ++ const int16_t* input, ++ int16_t* output, ++ size_t frames) { ++ if (!conversion.channel_map.empty()) { ++ const size_t output_channels = conversion.channel_map.size(); ++ for (size_t frame = 0; frame < frames; ++frame) { ++ for (size_t out = 0; out < output_channels; ++out) { ++ const int in = conversion.channel_map[out]; ++ *output++ = in < 0 ? 0 : input[in]; + } -+ *output++ = static_cast( -+ std::clamp(std::lrintf(sample), -32768l, 32767l)); ++ input += conversion.input_channels; + } -+ input += input_channels; ++ } else if (conversion.clipping) { ++ MixTerms(conversion, input, output, frames); ++ } else { ++ MixTerms(conversion, input, output, frames); + } +} + // An implementation of IMediaBuffer, as required for // IMediaObject::ProcessOutput(). After consuming data provided by // ProcessOutput(), call SetLength() to update the buffer availability. -@@ -1822,6 +1979,8 @@ +@@ -1822,6 +2046,8 @@ HRESULT hr = S_OK; WAVEFORMATEX* pWfxOut = NULL; WAVEFORMATEX Wfx = WAVEFORMATEX(); @@ -197,22 +264,21 @@ WAVEFORMATEX* pWfxClosestMatch = NULL; UINT bufferFrameCount = 0; constexpr int freqs[] = {48000, 44100, 16000, 96000, 32000, 8000}; -@@ -1901,6 +2060,8 @@ +@@ -1901,6 +2127,7 @@ // TODO(andrew): what happens in the event of failure in the above loop? // Is _ptrClientOut->Initialize expected to fail? // Same in InitRecording(). -+ _playMixMatrix.clear(); -+ _playChannelMap.clear(); ++ _playConversion = CoreAudioChannelConversion(); if (hr == S_OK) { _playAudioFrameSize = Wfx.nBlockAlign; // Block size is the number of samples each channel in 10ms. -@@ -1924,6 +2085,29 @@ +@@ -1924,6 +2151,29 @@ RTC_LOG(LS_VERBOSE) << "_playAudioFrameSize: " << _playAudioFrameSize; RTC_LOG(LS_VERBOSE) << "_playBlockSize : " << _playBlockSize; RTC_LOG(LS_VERBOSE) << "_playChannels : " << _playChannels; -+ } else if (pWfxOut && MixFormatForClient(pWfxOut, _playChannelsPrioList[0], -+ /*render=*/true, &WfxMix, -+ &_playMixMatrix, &_playChannelMap)) { ++ } else if (pWfxOut && ++ MixFormatForClient(pWfxOut, _playChannelsPrioList[0], ++ /*render=*/true, &WfxMix, &_playConversion)) { + // None of the formats above is supported. This happens when the mix + // format has more than two channels, as with gaming headsets that mix in + // 7.1, since a shared-mode stream must have the channel count of the mix @@ -236,7 +302,7 @@ } // Create a rendering stream. -@@ -1959,7 +2143,7 @@ +@@ -1959,7 +2209,7 @@ hnsBufferDuration, // requested buffer capacity as a time value (in // 100-nanosecond units) 0, // periodicity @@ -245,16 +311,15 @@ NULL); // session GUID if (FAILED(hr)) { -@@ -2056,6 +2240,8 @@ +@@ -2056,6 +2306,7 @@ ptrWav->cbSize = 0; // Set the VoE format equal to the AEC output format. -+ _recMixMatrix.clear(); -+ _recChannelMap.clear(); ++ _recConversion = CoreAudioChannelConversion(); _recAudioFrameSize = ptrWav->nBlockAlign; _recSampleRate = ptrWav->nSamplesPerSec; _recBlockSize = ptrWav->nSamplesPerSec / 100; -@@ -2136,6 +2322,8 @@ +@@ -2136,6 +2387,8 @@ HRESULT hr = S_OK; WAVEFORMATEX* pWfxIn = NULL; WAVEFORMATEXTENSIBLE Wfx = WAVEFORMATEXTENSIBLE(); @@ -263,22 +328,21 @@ WAVEFORMATEX* pWfxClosestMatch = NULL; UINT bufferFrameCount = 0; constexpr int freqs[6] = {48000, 44100, 16000, 96000, 32000, 8000}; -@@ -2216,6 +2404,8 @@ +@@ -2216,6 +2469,7 @@ break; } -+ _recMixMatrix.clear(); -+ _recChannelMap.clear(); ++ _recConversion = CoreAudioChannelConversion(); if (hr == S_OK) { _recAudioFrameSize = Wfx.Format.nBlockAlign; _recSampleRate = Wfx.Format.nSamplesPerSec; -@@ -2236,6 +2426,25 @@ +@@ -2236,6 +2490,25 @@ RTC_LOG(LS_VERBOSE) << "_recAudioFrameSize: " << _recAudioFrameSize; RTC_LOG(LS_VERBOSE) << "_recBlockSize : " << _recBlockSize; RTC_LOG(LS_VERBOSE) << "_recChannels : " << _recChannels; -+ } else if (pWfxIn && MixFormatForClient(pWfxIn, _recChannelsPrioList[0], -+ /*render=*/false, &WfxMix, -+ &_recMixMatrix, &_recChannelMap)) { ++ } else if (pWfxIn && ++ MixFormatForClient(pWfxIn, _recChannelsPrioList[0], ++ /*render=*/false, &WfxMix, &_recConversion)) { + // None of the formats above is supported; see InitPlayout(). Open the + // stream in the mix format and convert it to WebRTC's channel count in + // the capture thread. `_recAudioFrameSize` stays the size of a frame in @@ -298,7 +362,7 @@ } // Create a capturing stream. -@@ -2246,10 +2455,10 @@ +@@ -2246,10 +2519,10 @@ AUDCLNT_STREAMFLAGS_NOPERSIST, // volume and mute settings for an // audio session will not persist // across system restarts @@ -313,75 +377,88 @@ if (hr != S_OK) { RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; -@@ -2832,7 +3041,20 @@ +@@ -2832,7 +3105,14 @@ } // Get the actual (stored) data - nSamples = _ptrAudioBuffer->GetPlayoutData((int8_t*)pData); -+ if (_playMixMatrix.empty()) { ++ if (_playConversion.empty()) { + nSamples = _ptrAudioBuffer->GetPlayoutData((int8_t*)pData); + } else { + nSamples = + _ptrAudioBuffer->GetPlayoutData(_playConvertBuffer.data()); -+ if (!_playChannelMap.empty()) { -+ RemapChannels(_playChannelMap, _playChannels, -+ _playConvertBuffer.data(), ++ ConvertChannels(_playConversion, _playConvertBuffer.data(), + reinterpret_cast(pData), _playBlockSize); -+ } else { -+ MixChannels(_playMixMatrix, _playConvertBuffer.data(), -+ reinterpret_cast(pData), _playBlockSize); -+ } + } } DWORD dwFlags(0); -@@ -3221,7 +3443,18 @@ +@@ -3221,7 +3501,13 @@ RTC_DCHECK_NE(framesAvailable, 0); - if (pData) { -+ if (pData && !_recChannelMap.empty()) { -+ RemapChannels(_recChannelMap, _recMixMatrix[0].size(), -+ reinterpret_cast(pData), -+ reinterpret_cast( -+ &syncBuffer[syncBufIndex * _recAudioFrameSize]), -+ framesAvailable); -+ } else if (pData && !_recMixMatrix.empty()) { -+ MixChannels(_recMixMatrix, reinterpret_cast(pData), -+ reinterpret_cast( -+ &syncBuffer[syncBufIndex * _recAudioFrameSize]), -+ framesAvailable); ++ if (pData && !_recConversion.empty()) { ++ ConvertChannels(_recConversion, ++ reinterpret_cast(pData), ++ reinterpret_cast( ++ &syncBuffer[syncBufIndex * _recAudioFrameSize]), ++ framesAvailable); + } else if (pData) { CopyMemory(&syncBuffer[syncBufIndex * _recAudioFrameSize], pData, framesAvailable * _recAudioFrameSize); } else { --- a/modules/audio_device/win/audio_device_core_win.h +++ b/modules/audio_device/win/audio_device_core_win.h -@@ -21,6 +21,8 @@ +@@ -21,6 +21,9 @@ #include // IMediaObject #include // MMDevice ++#include +#include + #include "api/scoped_refptr.h" #include "modules/audio_device/audio_device_generic.h" #include "rtc_base/synchronization/mutex.h" -@@ -270,6 +272,20 @@ +@@ -43,6 +46,27 @@ + const uint16_t CORE_SPEAKER_VOLUME_STEP_SIZE = 1; + const uint16_t CORE_MICROPHONE_VOLUME_STEP_SIZE = 1; + ++// A conversion of interleaved 16-bit audio from one channel count to ++// another, built from a channel mixing matrix. A matrix that only routes ++// channels becomes a channel map, and the samples are copied, as Chrome's ++// ChannelMixer does. Any other matrix becomes the list of non-zero terms of ++// each output channel, with Q15 coefficients, as Firefox's cubeb mixer ++// applies it to 16-bit audio. ++struct CoreAudioChannelConversion { ++ // Channels per input frame; 0 when there is nothing to convert. ++ size_t input_channels = 0; ++ // For a matrix that only routes channels: the input channel of each output ++ // channel, or -1 for silence. Empty otherwise. ++ std::vector channel_map; ++ // For any other matrix: the (input channel, Q15 coefficient) pairs of each ++ // output channel. ++ std::vector>> terms; ++ // Whether a sum of terms can leave the 16-bit range, and must be clamped. ++ bool clipping = false; ++ ++ bool empty() const { return input_channels == 0; } ++}; ++ + class AudioDeviceWindowsCore : public AudioDeviceGeneric { + public: + AudioDeviceWindowsCore(); +@@ -270,6 +294,15 @@ uint16_t _recChannelsPrioList[3]; uint16_t _playChannelsPrioList[2]; + // Channel conversion for a device that is opened in its own mix format, -+ // because it accepts none of the formats above. Each matrix maps WebRTC's -+ // mono or stereo to the device's channels (render), or back (capture), and -+ // is empty when no conversion is needed. When a matrix only routes -+ // channels, the channel map names the input channel of each output channel -+ // (-1 for silence) and the samples are copied instead of mixed, as Chrome's -+ // ChannelMixer does; it is empty otherwise. `_playConvertBuffer` holds one -+ // 10 ms block in WebRTC's channel count before it is converted. -+ std::vector> _playMixMatrix; -+ std::vector> _recMixMatrix; -+ std::vector _playChannelMap; -+ std::vector _recChannelMap; ++ // because it accepts none of the formats above: from WebRTC's mono or ++ // stereo to the device's channels (render), or back (capture). Empty when ++ // no conversion is needed. `_playConvertBuffer` holds one 10 ms block in ++ // WebRTC's channel count before it is converted. ++ CoreAudioChannelConversion _playConversion; ++ CoreAudioChannelConversion _recConversion; + std::vector _playConvertBuffer; + LARGE_INTEGER _perfCounterFreq; From e99e0d5481d4c8c1a7bd2992e30ec0936da6d019 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 12 Oct 2026 17:41:26 +0200 Subject: [PATCH 4/4] fix: mix channels with 24-bit coefficients, which loses nothing to them The mix took Q15 coefficients from Firefox's cubeb mixer, which its 32-bit sums require. The sums here are 64-bit, so the coefficients now have 24 fractional bits. A float coefficient from 1/2 to 1, as all of WebRTC's downmix coefficients are, fits them exactly, and the sum of full-scale input over a row of the 7.1 downmix stays below 2^42. Against the exact mix, Q15 coefficients put 15 to 29% of the samples of a 7.1 or 5.1 downmix one or two off, at 1 to 5 dB less signal to noise than the float matrix had. With 24 bits, every sample of every case is the exact mix rounded half up, the float matrix is matched to the dB, and the speed is the same. Only the tie of a stereo to mono mix, half a step, rounds up rather than to even. --- .../patches/windows/wasapi_mix_format.patch | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch index 6fe92d3fe..0f8167d68 100644 --- a/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch +++ b/webrtc-jni/src/main/cpp/dependencies/webrtc/patches/windows/wasapi_mix_format.patch @@ -29,10 +29,17 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/platform_thread.h" -@@ -77,6 +82,225 @@ +@@ -77,6 +82,231 @@ enum { kAecCaptureStreamIndex = 0, kAecRenderStreamIndex = 1 }; ++// Fractional bits of the fixed-point coefficients of a channel mix. Firefox's ++// cubeb mixer takes 15, which its 32-bit sums require. Sums here are 64-bit, ++// so they can take 24: a float coefficient from 1/2 to 1 fits them exactly, ++// a smaller one to within 2^-25, and a row of WebRTC's 7.1 downmix, which ++// sums to about 6, keeps the sum of full-scale input below 2^42. ++constexpr int kCoefficientBits = 24; ++ +// Returns the channel layout of a WASAPI format, mapping the speaker masks +// the way Chrome's ChannelConfigToChannelLayout() does. A mask that is not +// one of these, or that does not match the channel count, is treated as @@ -149,26 +156,27 @@ + for (size_t out = 0; out < matrix.size(); ++out) { + int64_t sum = 0; + for (size_t in = 0; in < matrix[out].size(); ++in) { -+ const int32_t coefficient = -+ static_cast(std::lrintf(matrix[out][in] * 32768)); ++ const int32_t coefficient = static_cast( ++ std::lrintf(matrix[out][in] * (1 << kCoefficientBits))); + if (coefficient != 0) { + conversion->terms[out].emplace_back(in, coefficient); + sum += std::abs(coefficient); + } + } + // Full-scale input on every term would leave the 16-bit range. -+ if (sum > 32768) { ++ if (sum > (int64_t{1} << kCoefficientBits)) { + conversion->clipping = true; + } + } + return true; +} + -+// Rounds a sum of Q15 terms to a 16-bit sample, as Firefox's cubeb mixer -+// does, and clamps it when `kClamp`. ++// Rounds a sum of terms to a 16-bit sample, as Firefox's cubeb mixer does, ++// and clamps it when `kClamp`. +template -+int16_t RoundQ15(int64_t sum) { -+ int64_t sample = (sum + 16384) >> 15; ++int16_t RoundSum(int64_t sum) { ++ int64_t sample = ++ (sum + (int64_t{1} << (kCoefficientBits - 1))) >> kCoefficientBits; + if (kClamp) { + sample = std::clamp(sample, -32768, 32767); + } @@ -177,9 +185,7 @@ + +// Mixes `frames` interleaved frames with the terms of `conversion`, one output +// channel at a time and with loops for the usual numbers of terms, as -+// Firefox's cubeb mixer does. Sums are taken in 64 bits: cubeb's 32-bit sums -+// can overflow when several full-scale inputs mix into one channel, as in a -+// downmix of 7.1. ++// Firefox's cubeb mixer does. +template +void MixTerms(const CoreAudioChannelConversion& conversion, + const int16_t* input, @@ -201,7 +207,7 @@ + const int64_t ca = terms[0].second; + for (size_t frame = 0; frame < frames; ++frame) { + dest[frame * output_channels] = -+ RoundQ15(ca * a[frame * input_channels]); ++ RoundSum(ca * a[frame * input_channels]); + } + break; + } @@ -211,7 +217,7 @@ + const int64_t ca = terms[0].second; + const int64_t cb = terms[1].second; + for (size_t frame = 0; frame < frames; ++frame) { -+ dest[frame * output_channels] = RoundQ15( ++ dest[frame * output_channels] = RoundSum( + ca * a[frame * input_channels] + cb * b[frame * input_channels]); + } + break; @@ -223,7 +229,7 @@ + for (const auto& [channel, coefficient] : terms) { + sum += int64_t{coefficient} * in[channel]; + } -+ dest[frame * output_channels] = RoundQ15(sum); ++ dest[frame * output_channels] = RoundSum(sum); + } + break; + } @@ -255,7 +261,7 @@ // An implementation of IMediaBuffer, as required for // IMediaObject::ProcessOutput(). After consuming data provided by // ProcessOutput(), call SetLength() to update the buffer availability. -@@ -1822,6 +2046,8 @@ +@@ -1822,6 +2052,8 @@ HRESULT hr = S_OK; WAVEFORMATEX* pWfxOut = NULL; WAVEFORMATEX Wfx = WAVEFORMATEX(); @@ -264,7 +270,7 @@ WAVEFORMATEX* pWfxClosestMatch = NULL; UINT bufferFrameCount = 0; constexpr int freqs[] = {48000, 44100, 16000, 96000, 32000, 8000}; -@@ -1901,6 +2127,7 @@ +@@ -1901,6 +2133,7 @@ // TODO(andrew): what happens in the event of failure in the above loop? // Is _ptrClientOut->Initialize expected to fail? // Same in InitRecording(). @@ -272,7 +278,7 @@ if (hr == S_OK) { _playAudioFrameSize = Wfx.nBlockAlign; // Block size is the number of samples each channel in 10ms. -@@ -1924,6 +2151,29 @@ +@@ -1924,6 +2157,29 @@ RTC_LOG(LS_VERBOSE) << "_playAudioFrameSize: " << _playAudioFrameSize; RTC_LOG(LS_VERBOSE) << "_playBlockSize : " << _playBlockSize; RTC_LOG(LS_VERBOSE) << "_playChannels : " << _playChannels; @@ -302,7 +308,7 @@ } // Create a rendering stream. -@@ -1959,7 +2209,7 @@ +@@ -1959,7 +2215,7 @@ hnsBufferDuration, // requested buffer capacity as a time value (in // 100-nanosecond units) 0, // periodicity @@ -311,7 +317,7 @@ NULL); // session GUID if (FAILED(hr)) { -@@ -2056,6 +2306,7 @@ +@@ -2056,6 +2312,7 @@ ptrWav->cbSize = 0; // Set the VoE format equal to the AEC output format. @@ -319,7 +325,7 @@ _recAudioFrameSize = ptrWav->nBlockAlign; _recSampleRate = ptrWav->nSamplesPerSec; _recBlockSize = ptrWav->nSamplesPerSec / 100; -@@ -2136,6 +2387,8 @@ +@@ -2136,6 +2393,8 @@ HRESULT hr = S_OK; WAVEFORMATEX* pWfxIn = NULL; WAVEFORMATEXTENSIBLE Wfx = WAVEFORMATEXTENSIBLE(); @@ -328,7 +334,7 @@ WAVEFORMATEX* pWfxClosestMatch = NULL; UINT bufferFrameCount = 0; constexpr int freqs[6] = {48000, 44100, 16000, 96000, 32000, 8000}; -@@ -2216,6 +2469,7 @@ +@@ -2216,6 +2475,7 @@ break; } @@ -336,7 +342,7 @@ if (hr == S_OK) { _recAudioFrameSize = Wfx.Format.nBlockAlign; _recSampleRate = Wfx.Format.nSamplesPerSec; -@@ -2236,6 +2490,25 @@ +@@ -2236,6 +2496,25 @@ RTC_LOG(LS_VERBOSE) << "_recAudioFrameSize: " << _recAudioFrameSize; RTC_LOG(LS_VERBOSE) << "_recBlockSize : " << _recBlockSize; RTC_LOG(LS_VERBOSE) << "_recChannels : " << _recChannels; @@ -362,7 +368,7 @@ } // Create a capturing stream. -@@ -2246,10 +2519,10 @@ +@@ -2246,10 +2525,10 @@ AUDCLNT_STREAMFLAGS_NOPERSIST, // volume and mute settings for an // audio session will not persist // across system restarts @@ -377,7 +383,7 @@ if (hr != S_OK) { RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; -@@ -2832,7 +3105,14 @@ +@@ -2832,7 +3111,14 @@ } // Get the actual (stored) data @@ -393,7 +399,7 @@ } DWORD dwFlags(0); -@@ -3221,7 +3501,13 @@ +@@ -3221,7 +3507,13 @@ RTC_DCHECK_NE(framesAvailable, 0); @@ -428,7 +434,7 @@ +// another, built from a channel mixing matrix. A matrix that only routes +// channels becomes a channel map, and the samples are copied, as Chrome's +// ChannelMixer does. Any other matrix becomes the list of non-zero terms of -+// each output channel, with Q15 coefficients, as Firefox's cubeb mixer ++// each output channel, with integer coefficients, as Firefox's cubeb mixer +// applies it to 16-bit audio. +struct CoreAudioChannelConversion { + // Channels per input frame; 0 when there is nothing to convert. @@ -436,8 +442,8 @@ + // For a matrix that only routes channels: the input channel of each output + // channel, or -1 for silence. Empty otherwise. + std::vector channel_map; -+ // For any other matrix: the (input channel, Q15 coefficient) pairs of each -+ // output channel. ++ // For any other matrix: the (input channel, coefficient) pairs of each ++ // output channel, the coefficients in fixed point with 24 fractional bits. + std::vector>> terms; + // Whether a sum of terms can leave the 16-bit range, and must be clamped. + bool clipping = false;