From 62ab7c981d7f6e31aa2b2cc2cd0c296eb078f951 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 15:13:19 +0000 Subject: [PATCH 1/3] audio: webrtc_agc: add 32 kHz and 48 kHz sample rate support Extend webrtc_agc to accept 32 kHz and 48 kHz streams in addition to 8 kHz and 16 kHz. Downsample 32/48 kHz 10 ms frames to an anti-aliased 16 kHz signal via WebRtcSpl_DownsampleBy2 and WebRtcSpl_Resample48khzTo16khz for Band 0 VAD and adaptive gain analysis, while tracking full-rate 1 ms subframe peaks separately for the overload limiter. Interleave FIFO filling, 10 ms processing, and draining in chunks so calls larger than 10 ms never overflow the framing FIFOs. Update the webrtc-agc topology2 widget definition to list 8, 16, 32, and 48 kHz. Signed-off-by: Liam Girdwood --- src/audio/webrtc_agc/CMakeLists.txt | 1 + src/audio/webrtc_agc/README.md | 2 +- src/audio/webrtc_agc/llext/CMakeLists.txt | 2 + src/audio/webrtc_agc/webrtc_agc-agc.c | 330 +++++++++++++----- src/audio/webrtc_agc/webrtc_agc.c | 5 +- src/audio/webrtc_agc/webrtc_agc.h | 35 +- .../include/components/webrtc-agc.conf | 50 ++- 7 files changed, 331 insertions(+), 94 deletions(-) diff --git a/src/audio/webrtc_agc/CMakeLists.txt b/src/audio/webrtc_agc/CMakeLists.txt index 6417768293bd..cae9e1bf43fb 100644 --- a/src/audio/webrtc_agc/CMakeLists.txt +++ b/src/audio/webrtc_agc/CMakeLists.txt @@ -37,6 +37,7 @@ else() "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/division_operations.c" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_by_2_internal.c" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_fractional.c" + "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_48khz.c" ) set_source_files_properties( diff --git a/src/audio/webrtc_agc/README.md b/src/audio/webrtc_agc/README.md index 0120d6d3b853..5a5ff23cc587 100644 --- a/src/audio/webrtc_agc/README.md +++ b/src/audio/webrtc_agc/README.md @@ -8,7 +8,7 @@ Wraps WebRTC's digital Automatic Gain Control (`digital_agc.c`) behind the SOF ` - **Adaptive Speech Leveling**: Analyzes short-term speech envelope and applies dynamic digital gain to boost quiet talkers without amplifying background stationary noise. - **Integrated Saturation Limiter**: S-curve saturation limiter with fast lookahead attack and smooth release decay, guaranteeing zero digital overs. -- **Framing FIFO Engine**: Transparently decouples pipeline period sizes (e.g., 1 ms, 2 ms, or 5 ms) by buffering streaming audio into standard 10 ms WebRTC analysis frames (160 samples @ 16 kHz, 80 samples @ 8 kHz). +- **Framing FIFO Engine**: Transparently decouples pipeline period sizes (e.g., 1 ms, 2 ms, or 5 ms) by buffering streaming audio into standard 10 ms WebRTC analysis frames (80/160/320/480 samples @ 8/16/32/48 kHz). - **Pure Fixed-Point Implementation**: Pure integer Q15/Q31 arithmetic with zero floating-point unit dependencies (`CONFIG_FPU=n`). - **ALSA IPC4 Switch Control**: Runtime bypass/enable toggling via standard ALSA mixer controls (`SOF_IPC4_SWITCH_CONTROL_PARAM_ID`). - **Multichannel Support**: Accommodates up to 4 audio channels with per-channel state instances. diff --git a/src/audio/webrtc_agc/llext/CMakeLists.txt b/src/audio/webrtc_agc/llext/CMakeLists.txt index 196f7856f89d..17c94f8e7776 100644 --- a/src/audio/webrtc_agc/llext/CMakeLists.txt +++ b/src/audio/webrtc_agc/llext/CMakeLists.txt @@ -28,6 +28,8 @@ else() ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/dot_product_with_scale.c ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_by_2.c ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_by_2_internal.c + ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_fractional.c + ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_48khz.c INCLUDES "${SOF_WEBRTC_APM_SRC_DIR}" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/modules/audio_processing/agc/legacy" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/include" diff --git a/src/audio/webrtc_agc/webrtc_agc-agc.c b/src/audio/webrtc_agc/webrtc_agc-agc.c index 298bc1a22503..a493430b633d 100644 --- a/src/audio/webrtc_agc/webrtc_agc-agc.c +++ b/src/audio/webrtc_agc/webrtc_agc-agc.c @@ -6,7 +6,16 @@ */ #include "webrtc_agc.h" +#include #include +#include + +/* Overload limiter constants (matching digital_agc.c) using SOF Q-format macros */ +#define WEBRTC_AGC_PROBE_MAX_Q 13 +#define WEBRTC_AGC_PROBE_HEADROOM_Q15 Q_CONVERT_FLOAT(0.5, 15) +#define WEBRTC_AGC_LIMITER_SHIFT_DEFAULT 10 +#define WEBRTC_AGC_GAIN_NORM_TLD_Q16 Q_CONVERT_FLOAT(724.0773439, 16) +#define WEBRTC_AGC_LIMITER_DECAY_Q8 253 /* 253/256 (~-0.1 dB) in Q8 */ static int webrtc_agc_real_init(struct processing_module *mod) { @@ -18,10 +27,11 @@ static int webrtc_agc_real_prepare(struct processing_module *mod) struct webrtc_agc_comp_data *cd = module_get_private_data(mod); int c; - if (cd->rate != 8000 && cd->rate != 16000) + if (cd->rate != 8000 && cd->rate != 16000 && + cd->rate != 32000 && cd->rate != 48000) return -EINVAL; - cd->frame_samples = (cd->rate * 10) / 1000; + cd->frame_samples = (cd->rate * WEBRTC_AGC_FRAME_MS) / 1000; if (cd->frame_samples > WEBRTC_AGC_FRAME_SAMPLES_MAX) return -EINVAL; @@ -34,6 +44,8 @@ static int webrtc_agc_real_prepare(struct processing_module *mod) memset(ch->in_fifo, 0, sizeof(ch->in_fifo)); memset(ch->out_fifo, 0, sizeof(ch->out_fifo)); + memset(ch->down_state_32_to_16, 0, sizeof(ch->down_state_32_to_16)); + WebRtcSpl_ResetResample48khzTo16khz(&ch->state_48_to_16); WebRtcAgc_InitDigital(&ch->inst, cd->mode); WebRtcAgc_CalculateGainTable(ch->inst.gainTable, @@ -46,115 +58,260 @@ static int webrtc_agc_real_prepare(struct processing_module *mod) return 0; } +static void webrtc_agc_process_multiband(struct webrtc_agc_comp_data *cd, + struct webrtc_agc_channel *ch, + size_t num_bands, int16_t *dst) +{ + int16_t *vad_buf = cd->vad_buf; + int16_t *probe = cd->probe; + int32_t *gains = cd->gains; + const int16_t *in_ptrs[2]; + int16_t *out_ptrs[2]; + int32_t max_gain_q16; + int probe_shift; + int16_t probe_val; + size_t k, n, b; + + /* + * 2a. Anti-aliased 16 kHz speech-analysis downsampling for Band 0. + * WebRtcAgc_ProcessDigital() feeds Band 0 into WebRtcAgc_ProcessVad() + * to control adaptive gain decay and noise gating. Downsample the + * incoming 32 kHz or 48 kHz 10 ms frame to 16 kHz with WebRTC's + * low-pass filters so high-frequency content (> 8 kHz) does not alias + * into the VAD band. + */ + switch (cd->rate) { + case 32000: + WebRtcSpl_DownsampleBy2(ch->in_fifo, cd->frame_samples, vad_buf, + ch->down_state_32_to_16); + break; + case 48000: + default: + WebRtcSpl_Resample48khzTo16khz(ch->in_fifo, vad_buf, + &ch->state_48_to_16, + cd->resamp_tmpmem); + break; + } + + /* + * 2b. Run WebRtcAgc_ProcessDigital() at WEBRTC_AGC_BAND_RATE_HZ (16 kHz) + * on two synthetic 16 kHz sub-band vectors: the filtered vad_buf + * (Band 0, used for VAD and envelope tracking) and a constant probe + * vector (Band 1, used to extract the internal 10-subframe gain + * trajectory across WEBRTC_AGC_SUBFRAMES = 10 x 1 ms subframes). + * Bound the probe amplitude using the maximum of all gainTable + * entries and the current gain so the scaled int16_t probe output + * cannot overflow. + */ + max_gain_q16 = ch->inst.gain; + for (n = 0; n < ARRAY_SIZE(ch->inst.gainTable); n++) { + if (ch->inst.gainTable[n] > max_gain_q16) + max_gain_q16 = ch->inst.gainTable[n]; + } + + probe_shift = WEBRTC_AGC_PROBE_MAX_Q; + while (probe_shift > 0 && + Q_SHIFT(max_gain_q16, 16, probe_shift) > + WEBRTC_AGC_PROBE_HEADROOM_Q15) + probe_shift--; + probe_val = (int16_t)(1 << probe_shift); + + for (n = 0; n < WEBRTC_AGC_BAND_SAMPLES; n++) + probe[n] = probe_val; + + /* + * cd->gains has WEBRTC_AGC_GAIN_ENVELOPE_LEN (WEBRTC_AGC_SUBFRAMES + 1 = 11) + * elements: gains[0] holds the initial frame gain and gains[1..10] hold + * the end-of-subframe gains for subframes k = 0..9. + */ + gains[0] = ch->inst.gain; + in_ptrs[0] = vad_buf; + in_ptrs[1] = probe; + out_ptrs[0] = vad_buf; + out_ptrs[1] = probe; + + WebRtcAgc_ProcessDigital(&ch->inst, in_ptrs, ARRAY_SIZE(in_ptrs), + out_ptrs, WEBRTC_AGC_BAND_RATE_HZ, 0); + + /* + * Recover the Q16 gain at the start of each 1 ms subframe (10 subframes + * per 10 ms frame, each WEBRTC_AGC_SUBFRAME_BAND_SAMPLES = 16 samples + * long at the 16 kHz band rate). Index WEBRTC_AGC_SUBFRAMES (10) is the + * final element of gains[WEBRTC_AGC_GAIN_ENVELOPE_LEN] (size 11). + */ + for (k = 1; k < WEBRTC_AGC_SUBFRAMES; k++) + gains[k] = Q_SHIFT_LEFT((int32_t)probe[k * WEBRTC_AGC_SUBFRAME_BAND_SAMPLES], + probe_shift, 16); + gains[WEBRTC_AGC_SUBFRAMES] = ch->inst.gain; + + /* + * 2c. Separate full-rate overload limiter peak tracking. + * Measure the true 32/48 kHz peak energy across all samples of each + * 1 ms subframe (k = 0..9, where gains[k + 1] indexes gains[1..10] + * within gains[WEBRTC_AGC_GAIN_ENVELOPE_LEN]) and apply the WebRTC + * overload limiter (-0.1 dB step-down factor 253/256 and 1 ms + * lookahead) so high-frequency peaks attenuated in vad_buf cannot + * clip the full-rate output. + */ + for (k = 0; k < WEBRTC_AGC_SUBFRAMES; k++) { + size_t subfr_samples = WEBRTC_AGC_SUBFRAME_BAND_SAMPLES * num_bands; + const int16_t *subfr = &ch->in_fifo[k * subfr_samples]; + int32_t max_nrg = 0; + int16_t zeros = WEBRTC_AGC_LIMITER_SHIFT_DEFAULT; + int32_t gain32; + + for (n = 0; n < subfr_samples; n++) { + int32_t s = subfr[n]; + int32_t nrg = s * s; + + if (nrg > max_nrg) + max_nrg = nrg; + } + + /* k < WEBRTC_AGC_SUBFRAMES (10), so k + 1 <= 10 < WEBRTC_AGC_GAIN_ENVELOPE_LEN */ + if (gains[k + 1] > WEBRTC_AGC_GAIN_NORM_TLD_Q16) + zeros = 16 - WebRtcSpl_NormW32(gains[k + 1]); + + gain32 = (gains[k + 1] >> zeros) + 1; + gain32 *= gain32; + + while (AGC_MUL32(Q_SHIFT(max_nrg, 30, 18) + 1, gain32) > + WEBRTC_SPL_SHIFT_W32((int32_t)INT16_MAX, + 2 * (1 - zeros + WEBRTC_AGC_LIMITER_SHIFT_DEFAULT))) { + gains[k + 1] = (int32_t)Q_MULTS_32X32((int64_t)gains[k + 1], + WEBRTC_AGC_LIMITER_DECAY_Q8, + 16, 8, 16); + gain32 = (gains[k + 1] >> zeros) + 1; + gain32 *= gain32; + } + } + + for (k = 1; k < WEBRTC_AGC_SUBFRAMES; k++) { + if (gains[k] > gains[k + 1]) + gains[k] = gains[k + 1]; + } + ch->inst.gain = gains[WEBRTC_AGC_SUBFRAMES]; + + /* + * 2d. Apply the 10-subframe interpolated gain trajectory across all + * full-rate 32/48 kHz samples into dst using 64-bit Q-format + * multiplication before saturating to int16_t. + */ + for (k = 0; k < WEBRTC_AGC_SUBFRAMES; k++) { + int32_t delta = gains[k + 1] - gains[k]; + int32_t gain32 = Q_SHIFT_LEFT(gains[k], 16, 20); + + for (n = 0; n < WEBRTC_AGC_SUBFRAME_BAND_SAMPLES; n++) { + int32_t g = Q_SHIFT(gain32, 20, 16); + size_t base = (k * WEBRTC_AGC_SUBFRAME_BAND_SAMPLES + n) * num_bands; + + for (b = 0; b < num_bands; b++) { + int64_t tmp64 = Q_MULTS_32X32((int64_t)ch->in_fifo[base + b], + (int64_t)g, 15, 16, 15); + + dst[base + b] = sat_int16(sat_int32(tmp64)); + } + gain32 += delta; + } + } +} + static int webrtc_agc_real_process(struct processing_module *mod, const void *src, void *dst, int frames, enum sof_ipc_frame fmt) { struct webrtc_agc_comp_data *cd = module_get_private_data(mod); size_t ch_count = cd->channels; + size_t num_bands = (cd->rate <= WEBRTC_AGC_BAND_RATE_HZ) ? + 1 : (size_t)(cd->rate / WEBRTC_AGC_BAND_RATE_HZ); + size_t offset = 0; size_t c, i; if (frames <= 0) return 0; - /* 1. Pull input audio into planar in_fifo */ - { - size_t frames_to_copy = frames; - if (cd->in_fifo_count + frames_to_copy > WEBRTC_AGC_FIFO_FRAMES) - frames_to_copy = WEBRTC_AGC_FIFO_FRAMES - cd->in_fifo_count; - + /* + * Process in chunks of at most (frame_samples - in_fifo_count). + * Because out_fifo is pre-seeded with one 10 ms frame (frame_samples) + * of zeros in prepare()/reset(), the invariant + * (out_fifo_count - out_fifo_rd) == (frame_samples - in_fifo_count) + * holds at the start of every iteration: + * 1. Drain chunk frames from previously processed out_fifo data. + * 2. Enqueue chunk frames of new input into in_fifo. + * 3. When a full 10 ms frame is accumulated (in_fifo_count == + * frame_samples, which simultaneously exhausts out_fifo), process + * the 10 ms block into out_fifo[0..frame_samples - 1] and reset + * both FIFO cursors. + */ + while (offset < (size_t)frames) { + size_t rem = (size_t)frames - offset; + size_t space = (size_t)cd->frame_samples - cd->in_fifo_count; + size_t chunk = (rem < space) ? rem : space; + + /* 1. Drain chunk of processed frames from out_fifo to dst */ if (fmt == SOF_IPC_FRAME_S16_LE) { - const int16_t *s = src; - for (i = 0; i < frames_to_copy; i++) { + int16_t *d = (int16_t *)dst + offset * ch_count; + + for (i = 0; i < chunk; i++) { for (c = 0; c < ch_count; c++) - cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = s[i * ch_count + c]; + d[i * ch_count + c] = + cd->ch_data[c].out_fifo[cd->out_fifo_rd + i]; } } else { - const int32_t *s = src; - for (i = 0; i < frames_to_copy; i++) { + int32_t *d = (int32_t *)dst + offset * ch_count; + + for (i = 0; i < chunk; i++) { for (c = 0; c < ch_count; c++) - cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = (int16_t)(s[i * ch_count + c] >> 16); + d[i * ch_count + c] = + Q_SHIFT_LEFT((int32_t)cd->ch_data[c].out_fifo[cd->out_fifo_rd + i], + 15, 31); } } - cd->in_fifo_count += frames_to_copy; - } - - /* 2. Process complete 10 ms blocks */ - while (cd->in_fifo_count >= cd->frame_samples && - cd->out_fifo_count + cd->frame_samples <= WEBRTC_AGC_FIFO_FRAMES) { - int16_t proc_out[WEBRTC_AGC_FRAME_SAMPLES_MAX]; - - for (c = 0; c < ch_count; c++) { - struct webrtc_agc_channel *ch = &cd->ch_data[c]; - const int16_t *in_ptr = ch->in_fifo; - int16_t *out_ptr = proc_out; - - WebRtcAgc_ProcessDigital(&ch->inst, &in_ptr, 1, &out_ptr, - (cd->rate <= 16000) ? cd->rate : 16000, 0); - - /* Append to out_fifo */ - memcpy(&ch->out_fifo[cd->out_fifo_count], proc_out, - cd->frame_samples * sizeof(int16_t)); + cd->out_fifo_rd += chunk; - /* Shift remaining in_fifo */ - memmove(ch->in_fifo, &ch->in_fifo[cd->frame_samples], - (cd->in_fifo_count - cd->frame_samples) * sizeof(int16_t)); - } - cd->in_fifo_count -= cd->frame_samples; - cd->out_fifo_count += cd->frame_samples; - } + /* 2. Enqueue chunk of input frames into planar in_fifo */ + if (fmt == SOF_IPC_FRAME_S16_LE) { + const int16_t *s = (const int16_t *)src + offset * ch_count; - /* 3. Drain processed samples to dst buffer */ - { - size_t frames_avail = cd->out_fifo_count - cd->out_fifo_rd; - size_t frames_to_write = (frames_avail < (size_t)frames) ? frames_avail : (size_t)frames; - - if (frames_to_write > 0) { - if (fmt == SOF_IPC_FRAME_S16_LE) { - int16_t *d = dst; - for (i = 0; i < frames_to_write; i++) { - for (c = 0; c < ch_count; c++) - d[i * ch_count + c] = cd->ch_data[c].out_fifo[cd->out_fifo_rd + i]; - } - } else { - int32_t *d = dst; - for (i = 0; i < frames_to_write; i++) { - for (c = 0; c < ch_count; c++) - d[i * ch_count + c] = ((int32_t)cd->ch_data[c].out_fifo[cd->out_fifo_rd + i]) << 16; - } + for (i = 0; i < chunk; i++) { + for (c = 0; c < ch_count; c++) + cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = + s[i * ch_count + c]; } + } else { + const int32_t *s = (const int32_t *)src + offset * ch_count; - cd->out_fifo_rd += frames_to_write; - if (cd->out_fifo_rd == cd->out_fifo_count) { - cd->out_fifo_rd = 0; - cd->out_fifo_count = 0; - } else if (cd->out_fifo_rd > 0) { - size_t rem_fifo = cd->out_fifo_count - cd->out_fifo_rd; - - for (c = 0; c < ch_count; c++) { - memmove(cd->ch_data[c].out_fifo, - &cd->ch_data[c].out_fifo[cd->out_fifo_rd], - rem_fifo * sizeof(int16_t)); - } - cd->out_fifo_count = rem_fifo; - cd->out_fifo_rd = 0; + for (i = 0; i < chunk; i++) { + for (c = 0; c < ch_count; c++) + cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = + sat_int16(Q_SHIFT_RND(s[i * ch_count + c], + 31, 15)); } } - - /* If out_fifo had fewer frames than requested, zero-fill remainder */ - if (frames_to_write < (size_t)frames) { - size_t rem = (size_t)frames - frames_to_write; - - if (fmt == SOF_IPC_FRAME_S16_LE) { - int16_t *d = (int16_t *)dst + frames_to_write * ch_count; - - memset(d, 0, rem * ch_count * sizeof(int16_t)); - } else { - int32_t *d = (int32_t *)dst + frames_to_write * ch_count; - - memset(d, 0, rem * ch_count * sizeof(int32_t)); + cd->in_fifo_count += chunk; + + /* 3. Process a complete 10 ms block when in_fifo is full */ + if (cd->in_fifo_count == (size_t)cd->frame_samples) { + for (c = 0; c < ch_count; c++) { + struct webrtc_agc_channel *ch = &cd->ch_data[c]; + int16_t *out_ptr = ch->out_fifo; + + if (num_bands == 1) { + const int16_t *in_ptr = ch->in_fifo; + + WebRtcAgc_ProcessDigital(&ch->inst, &in_ptr, 1, + &out_ptr, cd->rate, 0); + } else { + webrtc_agc_process_multiband(cd, ch, + num_bands, out_ptr); + } } + cd->in_fifo_count = 0; + cd->out_fifo_rd = 0; + cd->out_fifo_count = cd->frame_samples; } + + offset += chunk; } return 0; @@ -172,6 +329,9 @@ static int webrtc_agc_real_reset(struct processing_module *mod) for (c = 0; c < cd->channels; c++) { memset(cd->ch_data[c].in_fifo, 0, sizeof(cd->ch_data[c].in_fifo)); memset(cd->ch_data[c].out_fifo, 0, sizeof(cd->ch_data[c].out_fifo)); + memset(cd->ch_data[c].down_state_32_to_16, 0, + sizeof(cd->ch_data[c].down_state_32_to_16)); + WebRtcSpl_ResetResample48khzTo16khz(&cd->ch_data[c].state_48_to_16); WebRtcAgc_InitDigital(&cd->ch_data[c].inst, cd->mode); WebRtcAgc_CalculateGainTable(cd->ch_data[c].inst.gainTable, cd->compression_gain_db, diff --git a/src/audio/webrtc_agc/webrtc_agc.c b/src/audio/webrtc_agc/webrtc_agc.c index 244d6b7a5a40..7576962196c8 100644 --- a/src/audio/webrtc_agc/webrtc_agc.c +++ b/src/audio/webrtc_agc/webrtc_agc.c @@ -76,8 +76,9 @@ __cold static int webrtc_agc_prepare(struct processing_module *mod, cd->rate = source_get_rate(sources[0]); cd->channels = source_get_channels(sources[0]); - if (cd->rate != 8000 && cd->rate != 16000) { - comp_err(dev, "webrtc_agc: unsupported rate %d (only 8000 and 16000 Hz supported)", + if (cd->rate != 8000 && cd->rate != 16000 && + cd->rate != 32000 && cd->rate != 48000) { + comp_err(dev, "webrtc_agc: unsupported rate %d (only 8/16/32/48 kHz supported)", cd->rate); return -EINVAL; } diff --git a/src/audio/webrtc_agc/webrtc_agc.h b/src/audio/webrtc_agc/webrtc_agc.h index 210aefea1154..b647d5f262ea 100644 --- a/src/audio/webrtc_agc/webrtc_agc.h +++ b/src/audio/webrtc_agc/webrtc_agc.h @@ -16,9 +16,26 @@ typedef struct { uint32_t dummy[64]; } DigitalAgc; #endif -#define WEBRTC_AGC_CHANNELS_MAX 4 -#define WEBRTC_AGC_FRAME_SAMPLES_MAX 480 /* 10 ms @ 48 kHz */ -#define WEBRTC_AGC_FIFO_FRAMES (WEBRTC_AGC_FRAME_SAMPLES_MAX * 2) +#define WEBRTC_AGC_CHANNELS_MAX 4 +/* + * WebRTC's WebRtcAgc_ProcessDigital() algorithm operates on a fixed 10 ms frame + * (10 x 1 ms subframes); the module's internal framing FIFO decouples this from + * the SOF pipeline period. + */ +#define WEBRTC_AGC_FRAME_MS 10 +/* 10 x 1 ms subframes per 10 ms frame; 11 envelope boundary points (0..10) */ +#define WEBRTC_AGC_SUBFRAMES 10 +#define WEBRTC_AGC_GAIN_ENVELOPE_LEN (WEBRTC_AGC_SUBFRAMES + 1) +#define WEBRTC_AGC_BAND_RATE_HZ 16000 +#define WEBRTC_AGC_MAX_RATE_HZ 48000 +#define WEBRTC_AGC_BAND_SAMPLES \ + ((WEBRTC_AGC_BAND_RATE_HZ * WEBRTC_AGC_FRAME_MS) / 1000) +#define WEBRTC_AGC_SUBFRAME_BAND_SAMPLES \ + (WEBRTC_AGC_BAND_SAMPLES / WEBRTC_AGC_SUBFRAMES) +#define WEBRTC_AGC_FRAME_SAMPLES_MAX \ + ((WEBRTC_AGC_MAX_RATE_HZ * WEBRTC_AGC_FRAME_MS) / 1000) +#define WEBRTC_AGC_FIFO_FRAMES WEBRTC_AGC_FRAME_SAMPLES_MAX +#define WEBRTC_AGC_RESAMP_TMPMEM_LEN (WEBRTC_AGC_FRAME_SAMPLES_MAX + 16) struct webrtc_agc_backend { const char *name; @@ -33,6 +50,10 @@ struct webrtc_agc_backend { struct webrtc_agc_channel { DigitalAgc inst; +#if !CONFIG_COMP_WEBRTC_AGC_STUB + int32_t down_state_32_to_16[8]; + WebRtcSpl_State48khzTo16khz state_48_to_16; +#endif int16_t in_fifo[WEBRTC_AGC_FIFO_FRAMES]; int16_t out_fifo[WEBRTC_AGC_FIFO_FRAMES]; }; @@ -41,6 +62,14 @@ struct webrtc_agc_comp_data { struct webrtc_agc_channel ch_data[WEBRTC_AGC_CHANNELS_MAX]; const struct webrtc_agc_backend *backend; +#if !CONFIG_COMP_WEBRTC_AGC_STUB + /* Scratch buffers for multi-band (32/48 kHz) processing */ + int16_t vad_buf[WEBRTC_AGC_BAND_SAMPLES]; + int16_t probe[WEBRTC_AGC_BAND_SAMPLES]; + int32_t gains[WEBRTC_AGC_GAIN_ENVELOPE_LEN]; + int32_t resamp_tmpmem[WEBRTC_AGC_RESAMP_TMPMEM_LEN]; +#endif + int channels; int rate; int frame_samples; /* samples per 10 ms frame */ diff --git a/tools/topology/topology2/include/components/webrtc-agc.conf b/tools/topology/topology2/include/components/webrtc-agc.conf index e0a2e9b684d5..9cc0ce4cf6ef 100644 --- a/tools/topology/topology2/include/components/webrtc-agc.conf +++ b/tools/topology/topology2/include/components/webrtc-agc.conf @@ -3,7 +3,7 @@ # # WebRTC Digital Automatic Gain Control widget (fixed-point). # Single input, single output. Dynamically normalizes speech levels (+9 dB boost, limiter). -# Supports 8 kHz and 16 kHz, S16 and S32 formats. +# Supports 8, 16, 32, and 48 kHz, S16 and S32 formats. # # Usage: # Object.Widget.webrtc-agc."N" { @@ -70,10 +70,34 @@ Class.Widget."webrtc-agc" { } } - num_input_audio_formats 4 - num_output_audio_formats 4 + num_input_audio_formats 8 + num_output_audio_formats 8 Object.Base.input_audio_format [ + # S16 stereo 48 kHz + { + in_rate 48000 + in_bit_depth 16 + in_valid_bit_depth 16 + } + # S32 stereo 48 kHz + { + in_rate 48000 + in_bit_depth 32 + in_valid_bit_depth 32 + } + # S16 stereo 32 kHz + { + in_rate 32000 + in_bit_depth 16 + in_valid_bit_depth 16 + } + # S32 stereo 32 kHz + { + in_rate 32000 + in_bit_depth 32 + in_valid_bit_depth 32 + } # S16 stereo 16 kHz { in_rate 16000 @@ -101,6 +125,26 @@ Class.Widget."webrtc-agc" { ] Object.Base.output_audio_format [ + { + out_rate 48000 + out_bit_depth 16 + out_valid_bit_depth 16 + } + { + out_rate 48000 + out_bit_depth 32 + out_valid_bit_depth 32 + } + { + out_rate 32000 + out_bit_depth 16 + out_valid_bit_depth 16 + } + { + out_rate 32000 + out_bit_depth 32 + out_valid_bit_depth 32 + } { out_rate 16000 out_bit_depth 16 From 7aeb3692928a618d78e1794fb29b6bec9666d78d Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 17:11:11 +0000 Subject: [PATCH 2/3] audio: webrtc_aec: configure module for DP domain scheduling Configure the webrtc_aec LLEXT manifest (.domain_dp = 1), rimage TOML (domain_types = "1"), and topology2 widget definition (scheduler_domain "DP" with stack_bytes_requirement 16384) to run in the Data Processing (DP) scheduling domain rather than on the 1 ms Low Latency (LL) thread. Add webrtc_aec_is_ready_to_process() so the DP scheduler triggers processing whenever a full 10 ms microphone capture frame is available and output space is free, without blocking when the playback echo reference stream is idle. Signed-off-by: Liam Girdwood --- src/audio/webrtc_aec/webrtc_aec.c | 51 +++++++++++++++---- src/audio/webrtc_aec/webrtc_aec.toml | 2 +- .../include/components/webrtc-aec.conf | 11 ++-- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/audio/webrtc_aec/webrtc_aec.c b/src/audio/webrtc_aec/webrtc_aec.c index 74e6f771cb34..4c3f8f027a27 100644 --- a/src/audio/webrtc_aec/webrtc_aec.c +++ b/src/audio/webrtc_aec/webrtc_aec.c @@ -641,14 +641,35 @@ static int webrtc_aec_get_config(struct processing_module *mod, uint32_t config_ return -EINVAL; } +static bool webrtc_aec_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, + int num_of_sources, + struct sof_sink **sinks, + int num_of_sinks) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct sof_source *mic; + struct sof_sink *out; + + if (!cd || !cd->configured || num_of_sources <= cd->mic_src || num_of_sinks < 1) + return false; + + mic = sources[cd->mic_src]; + out = sinks[0]; + + return source_get_data_available(mic) >= source_get_min_available(mic) && + sink_get_free_size(out) >= sink_get_min_free_space(out); +} + static const struct module_interface webrtc_aec_interface = { - .init = webrtc_aec_init, - .prepare = webrtc_aec_prepare, - .process = webrtc_aec_process, - .set_configuration = webrtc_aec_set_config, - .get_configuration = webrtc_aec_get_config, - .reset = webrtc_aec_reset, - .free = webrtc_aec_free, + .init = webrtc_aec_init, + .prepare = webrtc_aec_prepare, + .is_ready_to_process = webrtc_aec_is_ready_to_process, + .process = webrtc_aec_process, + .set_configuration = webrtc_aec_set_config, + .get_configuration = webrtc_aec_get_config, + .reset = webrtc_aec_reset, + .free = webrtc_aec_free, }; #if CONFIG_COMP_WEBRTC_AEC_MODULE @@ -657,9 +678,19 @@ static const struct module_interface webrtc_aec_interface = { #include #include -static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("WRTCAEC", &webrtc_aec_interface, 2, - SOF_REG_UUID(webrtc_aec), 40); +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = { + .module = { + .name = "WRTCAEC", + .uuid = SOF_REG_UUID(webrtc_aec), + .entry_point = (uint32_t)(&webrtc_aec_interface), + .instance_max_count = 40, + .type = { + .load_type = SOF_MAN_MOD_TYPE_LLEXT, + .domain_dp = 1, + }, + .affinity_mask = 2, + } +}; SOF_LLEXT_BUILDINFO; diff --git a/src/audio/webrtc_aec/webrtc_aec.toml b/src/audio/webrtc_aec/webrtc_aec.toml index fde0f102e0ce..a6f10aa3cb4c 100644 --- a/src/audio/webrtc_aec/webrtc_aec.toml +++ b/src/audio/webrtc_aec/webrtc_aec.toml @@ -9,7 +9,7 @@ uuid = UUIDREG_STR_WEBRTC_AEC affinity_mask = "0x1" instance_count = "40" - domain_types = "0" + domain_types = "1" load_type = LOAD_TYPE module_type = "9" auto_start = "0" diff --git a/tools/topology/topology2/include/components/webrtc-aec.conf b/tools/topology/topology2/include/components/webrtc-aec.conf index f83f8cd5d25e..f90cd3d8f108 100644 --- a/tools/topology/topology2/include/components/webrtc-aec.conf +++ b/tools/topology/topology2/include/components/webrtc-aec.conf @@ -61,6 +61,7 @@ Class.Widget."webrtc-aec" { !immutable [ "uuid" "type" + "scheduler_domain" ] !deprecated [ @@ -135,10 +136,12 @@ Class.Widget."webrtc-aec" { ] # UUID: e5da7b5b-133a-ba46-b517651d6300bb83 - uuid "5b:7b:da:e5:3a:13:46:ba:b5:17:65:1d:63:00:bb:83" - type "effect" - no_pm "true" - cpc 2000 + uuid "5b:7b:da:e5:3a:13:46:ba:b5:17:65:1d:63:00:bb:83" + type "effect" + no_pm "true" + cpc 2000 + scheduler_domain "DP" + stack_bytes_requirement 16384 num_input_pins 2 num_output_pins 1 } From 1bf9c2043f1ab8f06687a154e8bd3146337b6a0c Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 15:15:26 +0000 Subject: [PATCH 3/3] app: add WebRTC Kconfig overlays and COMP_STUBS defaults Select the pass-through stub backends by default for webrtc_aec, webrtc_agc, and webrtc_hpf when CONFIG_COMP_STUBS=y (matching webrtc_ns, webrtc_ns2, and webrtc_vad), and include all six WebRTC modules as LLEXT modules (=m) in app/stub_build_all_ipc4.conf. Add app/webrtc_overlay.conf (LLEXT real backends + heap sizing) and app/webrtc_stubs_overlay.conf (LLEXT stub backends) for firmware builds. Signed-off-by: Liam Girdwood --- app/stub_build_all_ipc4.conf | 6 +++++ app/webrtc_overlay.conf | 44 +++++++++++++++++++++++++++++++++++ app/webrtc_stubs_overlay.conf | 16 +++++++++++++ src/audio/webrtc_aec/Kconfig | 1 + src/audio/webrtc_agc/Kconfig | 2 +- src/audio/webrtc_hpf/Kconfig | 2 +- 6 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 app/webrtc_overlay.conf create mode 100644 app/webrtc_stubs_overlay.conf diff --git a/app/stub_build_all_ipc4.conf b/app/stub_build_all_ipc4.conf index 6fdbd7209e47..5f46e458c94c 100644 --- a/app/stub_build_all_ipc4.conf +++ b/app/stub_build_all_ipc4.conf @@ -2,6 +2,12 @@ CONFIG_COMP_STUBS=y CONFIG_COMP_ARIA=y CONFIG_COMP_CHAIN_DMA=y CONFIG_COMP_UP_DOWN_MIXER=y +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_AGC=m +CONFIG_COMP_WEBRTC_HPF=m CONFIG_IPC_MAJOR_4=y CONFIG_SAMPLES=y CONFIG_SAMPLE_SMART_AMP=y diff --git a/app/webrtc_overlay.conf b/app/webrtc_overlay.conf new file mode 100644 index 000000000000..c51897102502 --- /dev/null +++ b/app/webrtc_overlay.conf @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# WebRTC Audio Processing Modules Kconfig overlay (LLEXT real backends) + +# WebRTC NS (classic spectral Wiener filter) +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_NS_STUB=n +CONFIG_WEBRTC_NS_LEVEL=2 +CONFIG_WEBRTC_NS_SAMPLE_RATE_HZ=16000 + +# WebRTC VAD (libfvad Voice Activity Detector) +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_VAD_STUB=n +CONFIG_WEBRTC_VAD_MODE=2 +CONFIG_WEBRTC_VAD_FRAME_MS=10 + +# WebRTC AEC (Acoustic Echo Canceller) +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_WEBRTC_AEC_BACKEND_AECM=y +CONFIG_WEBRTC_AEC_FILTER_LEN_MS=64 +CONFIG_WEBRTC_AEC_SUPPRESSION_LEVEL=1 +CONFIG_WEBRTC_AEC_SAMPLE_RATE_HZ=16000 + +# WebRTC NS2 (RNNoise neural network noise suppression) +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_NS2_STUB=n +CONFIG_WEBRTC_NS2_CHANNELS_MAX=2 +CONFIG_WEBRTC_NS2_VAD_THRESHOLD_PCT=60 +CONFIG_WEBRTC_NS2_VAD_NOTIFY=y + +# WebRTC HPF (High-Pass Filter 80 Hz biquad) +CONFIG_COMP_WEBRTC_HPF=m +CONFIG_COMP_WEBRTC_HPF_STUB=n + +# WebRTC AGC (Automatic Gain Control & Limiter) +CONFIG_COMP_WEBRTC_AGC=m +CONFIG_COMP_WEBRTC_AGC_STUB=n +CONFIG_WEBRTC_AGC_TARGET_DBFS=3 +CONFIG_WEBRTC_AGC_COMPRESSION_GAIN_DB=9 +CONFIG_WEBRTC_AGC_LIMITER=y + +# Heap and stack sizing for full WebRTC suite +CONFIG_HEAP_MEM_POOL_SIZE=131072 +CONFIG_SOF_ZEPHYR_HEAP_SIZE=0x80000 diff --git a/app/webrtc_stubs_overlay.conf b/app/webrtc_stubs_overlay.conf new file mode 100644 index 000000000000..eb51d1b892d4 --- /dev/null +++ b/app/webrtc_stubs_overlay.conf @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# WebRTC Audio Processing Modules Kconfig overlay (LLEXT pass-through stub backends) + +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_VAD_STUB=y +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_NS_STUB=y +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_COMP_WEBRTC_AEC_STUB=y +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_NS2_STUB=y +CONFIG_COMP_WEBRTC_AGC=m +CONFIG_COMP_WEBRTC_AGC_STUB=y +CONFIG_COMP_WEBRTC_HPF=m +CONFIG_COMP_WEBRTC_HPF_STUB=y diff --git a/src/audio/webrtc_aec/Kconfig b/src/audio/webrtc_aec/Kconfig index 067ddb58d065..9d8a78b2b2ae 100644 --- a/src/audio/webrtc_aec/Kconfig +++ b/src/audio/webrtc_aec/Kconfig @@ -26,6 +26,7 @@ if COMP_WEBRTC_AEC choice WEBRTC_AEC_BACKEND prompt "WebRTC AEC processing backend" + default COMP_WEBRTC_AEC_STUB if COMP_STUBS default WEBRTC_AEC_BACKEND_FLOAT if COMP_WEBRTC_AEC = y default WEBRTC_AEC_BACKEND_AECM diff --git a/src/audio/webrtc_agc/Kconfig b/src/audio/webrtc_agc/Kconfig index 1e2246a74513..cf9effa0104a 100644 --- a/src/audio/webrtc_agc/Kconfig +++ b/src/audio/webrtc_agc/Kconfig @@ -11,7 +11,7 @@ if COMP_WEBRTC_AGC config COMP_WEBRTC_AGC_STUB bool "Build with pass-through stub backend" - default n + default y if COMP_STUBS help Select to build with the pass-through stub backend instead of the real AGC. diff --git a/src/audio/webrtc_hpf/Kconfig b/src/audio/webrtc_hpf/Kconfig index 07a5623984ed..6d6d57a2e728 100644 --- a/src/audio/webrtc_hpf/Kconfig +++ b/src/audio/webrtc_hpf/Kconfig @@ -11,7 +11,7 @@ if COMP_WEBRTC_HPF config COMP_WEBRTC_HPF_STUB bool "Build with pass-through stub backend" - default n + default y if COMP_STUBS help Select to build with the pass-through stub backend instead of the real filter.