diff --git a/app/stub_build_all_ipc4.conf b/app/stub_build_all_ipc4.conf index 6fdbd7209e47..5f46e458c94c 100644 --- a/app/stub_build_all_ipc4.conf +++ b/app/stub_build_all_ipc4.conf @@ -2,6 +2,12 @@ CONFIG_COMP_STUBS=y CONFIG_COMP_ARIA=y CONFIG_COMP_CHAIN_DMA=y CONFIG_COMP_UP_DOWN_MIXER=y +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_AGC=m +CONFIG_COMP_WEBRTC_HPF=m CONFIG_IPC_MAJOR_4=y CONFIG_SAMPLES=y CONFIG_SAMPLE_SMART_AMP=y diff --git a/app/webrtc_overlay.conf b/app/webrtc_overlay.conf new file mode 100644 index 000000000000..c51897102502 --- /dev/null +++ b/app/webrtc_overlay.conf @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# WebRTC Audio Processing Modules Kconfig overlay (LLEXT real backends) + +# WebRTC NS (classic spectral Wiener filter) +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_NS_STUB=n +CONFIG_WEBRTC_NS_LEVEL=2 +CONFIG_WEBRTC_NS_SAMPLE_RATE_HZ=16000 + +# WebRTC VAD (libfvad Voice Activity Detector) +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_VAD_STUB=n +CONFIG_WEBRTC_VAD_MODE=2 +CONFIG_WEBRTC_VAD_FRAME_MS=10 + +# WebRTC AEC (Acoustic Echo Canceller) +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_WEBRTC_AEC_BACKEND_AECM=y +CONFIG_WEBRTC_AEC_FILTER_LEN_MS=64 +CONFIG_WEBRTC_AEC_SUPPRESSION_LEVEL=1 +CONFIG_WEBRTC_AEC_SAMPLE_RATE_HZ=16000 + +# WebRTC NS2 (RNNoise neural network noise suppression) +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_NS2_STUB=n +CONFIG_WEBRTC_NS2_CHANNELS_MAX=2 +CONFIG_WEBRTC_NS2_VAD_THRESHOLD_PCT=60 +CONFIG_WEBRTC_NS2_VAD_NOTIFY=y + +# WebRTC HPF (High-Pass Filter 80 Hz biquad) +CONFIG_COMP_WEBRTC_HPF=m +CONFIG_COMP_WEBRTC_HPF_STUB=n + +# WebRTC AGC (Automatic Gain Control & Limiter) +CONFIG_COMP_WEBRTC_AGC=m +CONFIG_COMP_WEBRTC_AGC_STUB=n +CONFIG_WEBRTC_AGC_TARGET_DBFS=3 +CONFIG_WEBRTC_AGC_COMPRESSION_GAIN_DB=9 +CONFIG_WEBRTC_AGC_LIMITER=y + +# Heap and stack sizing for full WebRTC suite +CONFIG_HEAP_MEM_POOL_SIZE=131072 +CONFIG_SOF_ZEPHYR_HEAP_SIZE=0x80000 diff --git a/app/webrtc_stubs_overlay.conf b/app/webrtc_stubs_overlay.conf new file mode 100644 index 000000000000..eb51d1b892d4 --- /dev/null +++ b/app/webrtc_stubs_overlay.conf @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# WebRTC Audio Processing Modules Kconfig overlay (LLEXT pass-through stub backends) + +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_VAD_STUB=y +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_NS_STUB=y +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_COMP_WEBRTC_AEC_STUB=y +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_NS2_STUB=y +CONFIG_COMP_WEBRTC_AGC=m +CONFIG_COMP_WEBRTC_AGC_STUB=y +CONFIG_COMP_WEBRTC_HPF=m +CONFIG_COMP_WEBRTC_HPF_STUB=y diff --git a/src/audio/webrtc_aec/Kconfig b/src/audio/webrtc_aec/Kconfig index 067ddb58d065..9d8a78b2b2ae 100644 --- a/src/audio/webrtc_aec/Kconfig +++ b/src/audio/webrtc_aec/Kconfig @@ -26,6 +26,7 @@ if COMP_WEBRTC_AEC choice WEBRTC_AEC_BACKEND prompt "WebRTC AEC processing backend" + default COMP_WEBRTC_AEC_STUB if COMP_STUBS default WEBRTC_AEC_BACKEND_FLOAT if COMP_WEBRTC_AEC = y default WEBRTC_AEC_BACKEND_AECM diff --git a/src/audio/webrtc_aec/webrtc_aec.c b/src/audio/webrtc_aec/webrtc_aec.c index 74e6f771cb34..4c3f8f027a27 100644 --- a/src/audio/webrtc_aec/webrtc_aec.c +++ b/src/audio/webrtc_aec/webrtc_aec.c @@ -641,14 +641,35 @@ static int webrtc_aec_get_config(struct processing_module *mod, uint32_t config_ return -EINVAL; } +static bool webrtc_aec_is_ready_to_process(struct processing_module *mod, + struct sof_source **sources, + int num_of_sources, + struct sof_sink **sinks, + int num_of_sinks) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct sof_source *mic; + struct sof_sink *out; + + if (!cd || !cd->configured || num_of_sources <= cd->mic_src || num_of_sinks < 1) + return false; + + mic = sources[cd->mic_src]; + out = sinks[0]; + + return source_get_data_available(mic) >= source_get_min_available(mic) && + sink_get_free_size(out) >= sink_get_min_free_space(out); +} + static const struct module_interface webrtc_aec_interface = { - .init = webrtc_aec_init, - .prepare = webrtc_aec_prepare, - .process = webrtc_aec_process, - .set_configuration = webrtc_aec_set_config, - .get_configuration = webrtc_aec_get_config, - .reset = webrtc_aec_reset, - .free = webrtc_aec_free, + .init = webrtc_aec_init, + .prepare = webrtc_aec_prepare, + .is_ready_to_process = webrtc_aec_is_ready_to_process, + .process = webrtc_aec_process, + .set_configuration = webrtc_aec_set_config, + .get_configuration = webrtc_aec_get_config, + .reset = webrtc_aec_reset, + .free = webrtc_aec_free, }; #if CONFIG_COMP_WEBRTC_AEC_MODULE @@ -657,9 +678,19 @@ static const struct module_interface webrtc_aec_interface = { #include #include -static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("WRTCAEC", &webrtc_aec_interface, 2, - SOF_REG_UUID(webrtc_aec), 40); +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = { + .module = { + .name = "WRTCAEC", + .uuid = SOF_REG_UUID(webrtc_aec), + .entry_point = (uint32_t)(&webrtc_aec_interface), + .instance_max_count = 40, + .type = { + .load_type = SOF_MAN_MOD_TYPE_LLEXT, + .domain_dp = 1, + }, + .affinity_mask = 2, + } +}; SOF_LLEXT_BUILDINFO; diff --git a/src/audio/webrtc_aec/webrtc_aec.toml b/src/audio/webrtc_aec/webrtc_aec.toml index fde0f102e0ce..a6f10aa3cb4c 100644 --- a/src/audio/webrtc_aec/webrtc_aec.toml +++ b/src/audio/webrtc_aec/webrtc_aec.toml @@ -9,7 +9,7 @@ uuid = UUIDREG_STR_WEBRTC_AEC affinity_mask = "0x1" instance_count = "40" - domain_types = "0" + domain_types = "1" load_type = LOAD_TYPE module_type = "9" auto_start = "0" diff --git a/src/audio/webrtc_agc/CMakeLists.txt b/src/audio/webrtc_agc/CMakeLists.txt index 6417768293bd..cae9e1bf43fb 100644 --- a/src/audio/webrtc_agc/CMakeLists.txt +++ b/src/audio/webrtc_agc/CMakeLists.txt @@ -37,6 +37,7 @@ else() "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/division_operations.c" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_by_2_internal.c" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_fractional.c" + "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_48khz.c" ) set_source_files_properties( diff --git a/src/audio/webrtc_agc/Kconfig b/src/audio/webrtc_agc/Kconfig index 1e2246a74513..cf9effa0104a 100644 --- a/src/audio/webrtc_agc/Kconfig +++ b/src/audio/webrtc_agc/Kconfig @@ -11,7 +11,7 @@ if COMP_WEBRTC_AGC config COMP_WEBRTC_AGC_STUB bool "Build with pass-through stub backend" - default n + default y if COMP_STUBS help Select to build with the pass-through stub backend instead of the real AGC. diff --git a/src/audio/webrtc_agc/README.md b/src/audio/webrtc_agc/README.md index 0120d6d3b853..5a5ff23cc587 100644 --- a/src/audio/webrtc_agc/README.md +++ b/src/audio/webrtc_agc/README.md @@ -8,7 +8,7 @@ Wraps WebRTC's digital Automatic Gain Control (`digital_agc.c`) behind the SOF ` - **Adaptive Speech Leveling**: Analyzes short-term speech envelope and applies dynamic digital gain to boost quiet talkers without amplifying background stationary noise. - **Integrated Saturation Limiter**: S-curve saturation limiter with fast lookahead attack and smooth release decay, guaranteeing zero digital overs. -- **Framing FIFO Engine**: Transparently decouples pipeline period sizes (e.g., 1 ms, 2 ms, or 5 ms) by buffering streaming audio into standard 10 ms WebRTC analysis frames (160 samples @ 16 kHz, 80 samples @ 8 kHz). +- **Framing FIFO Engine**: Transparently decouples pipeline period sizes (e.g., 1 ms, 2 ms, or 5 ms) by buffering streaming audio into standard 10 ms WebRTC analysis frames (80/160/320/480 samples @ 8/16/32/48 kHz). - **Pure Fixed-Point Implementation**: Pure integer Q15/Q31 arithmetic with zero floating-point unit dependencies (`CONFIG_FPU=n`). - **ALSA IPC4 Switch Control**: Runtime bypass/enable toggling via standard ALSA mixer controls (`SOF_IPC4_SWITCH_CONTROL_PARAM_ID`). - **Multichannel Support**: Accommodates up to 4 audio channels with per-channel state instances. diff --git a/src/audio/webrtc_agc/llext/CMakeLists.txt b/src/audio/webrtc_agc/llext/CMakeLists.txt index 196f7856f89d..17c94f8e7776 100644 --- a/src/audio/webrtc_agc/llext/CMakeLists.txt +++ b/src/audio/webrtc_agc/llext/CMakeLists.txt @@ -28,6 +28,8 @@ else() ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/dot_product_with_scale.c ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_by_2.c ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_by_2_internal.c + ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_fractional.c + ${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/resample_48khz.c INCLUDES "${SOF_WEBRTC_APM_SRC_DIR}" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/modules/audio_processing/agc/legacy" "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/common_audio/signal_processing/include" diff --git a/src/audio/webrtc_agc/webrtc_agc-agc.c b/src/audio/webrtc_agc/webrtc_agc-agc.c index 298bc1a22503..a493430b633d 100644 --- a/src/audio/webrtc_agc/webrtc_agc-agc.c +++ b/src/audio/webrtc_agc/webrtc_agc-agc.c @@ -6,7 +6,16 @@ */ #include "webrtc_agc.h" +#include #include +#include + +/* Overload limiter constants (matching digital_agc.c) using SOF Q-format macros */ +#define WEBRTC_AGC_PROBE_MAX_Q 13 +#define WEBRTC_AGC_PROBE_HEADROOM_Q15 Q_CONVERT_FLOAT(0.5, 15) +#define WEBRTC_AGC_LIMITER_SHIFT_DEFAULT 10 +#define WEBRTC_AGC_GAIN_NORM_TLD_Q16 Q_CONVERT_FLOAT(724.0773439, 16) +#define WEBRTC_AGC_LIMITER_DECAY_Q8 253 /* 253/256 (~-0.1 dB) in Q8 */ static int webrtc_agc_real_init(struct processing_module *mod) { @@ -18,10 +27,11 @@ static int webrtc_agc_real_prepare(struct processing_module *mod) struct webrtc_agc_comp_data *cd = module_get_private_data(mod); int c; - if (cd->rate != 8000 && cd->rate != 16000) + if (cd->rate != 8000 && cd->rate != 16000 && + cd->rate != 32000 && cd->rate != 48000) return -EINVAL; - cd->frame_samples = (cd->rate * 10) / 1000; + cd->frame_samples = (cd->rate * WEBRTC_AGC_FRAME_MS) / 1000; if (cd->frame_samples > WEBRTC_AGC_FRAME_SAMPLES_MAX) return -EINVAL; @@ -34,6 +44,8 @@ static int webrtc_agc_real_prepare(struct processing_module *mod) memset(ch->in_fifo, 0, sizeof(ch->in_fifo)); memset(ch->out_fifo, 0, sizeof(ch->out_fifo)); + memset(ch->down_state_32_to_16, 0, sizeof(ch->down_state_32_to_16)); + WebRtcSpl_ResetResample48khzTo16khz(&ch->state_48_to_16); WebRtcAgc_InitDigital(&ch->inst, cd->mode); WebRtcAgc_CalculateGainTable(ch->inst.gainTable, @@ -46,115 +58,260 @@ static int webrtc_agc_real_prepare(struct processing_module *mod) return 0; } +static void webrtc_agc_process_multiband(struct webrtc_agc_comp_data *cd, + struct webrtc_agc_channel *ch, + size_t num_bands, int16_t *dst) +{ + int16_t *vad_buf = cd->vad_buf; + int16_t *probe = cd->probe; + int32_t *gains = cd->gains; + const int16_t *in_ptrs[2]; + int16_t *out_ptrs[2]; + int32_t max_gain_q16; + int probe_shift; + int16_t probe_val; + size_t k, n, b; + + /* + * 2a. Anti-aliased 16 kHz speech-analysis downsampling for Band 0. + * WebRtcAgc_ProcessDigital() feeds Band 0 into WebRtcAgc_ProcessVad() + * to control adaptive gain decay and noise gating. Downsample the + * incoming 32 kHz or 48 kHz 10 ms frame to 16 kHz with WebRTC's + * low-pass filters so high-frequency content (> 8 kHz) does not alias + * into the VAD band. + */ + switch (cd->rate) { + case 32000: + WebRtcSpl_DownsampleBy2(ch->in_fifo, cd->frame_samples, vad_buf, + ch->down_state_32_to_16); + break; + case 48000: + default: + WebRtcSpl_Resample48khzTo16khz(ch->in_fifo, vad_buf, + &ch->state_48_to_16, + cd->resamp_tmpmem); + break; + } + + /* + * 2b. Run WebRtcAgc_ProcessDigital() at WEBRTC_AGC_BAND_RATE_HZ (16 kHz) + * on two synthetic 16 kHz sub-band vectors: the filtered vad_buf + * (Band 0, used for VAD and envelope tracking) and a constant probe + * vector (Band 1, used to extract the internal 10-subframe gain + * trajectory across WEBRTC_AGC_SUBFRAMES = 10 x 1 ms subframes). + * Bound the probe amplitude using the maximum of all gainTable + * entries and the current gain so the scaled int16_t probe output + * cannot overflow. + */ + max_gain_q16 = ch->inst.gain; + for (n = 0; n < ARRAY_SIZE(ch->inst.gainTable); n++) { + if (ch->inst.gainTable[n] > max_gain_q16) + max_gain_q16 = ch->inst.gainTable[n]; + } + + probe_shift = WEBRTC_AGC_PROBE_MAX_Q; + while (probe_shift > 0 && + Q_SHIFT(max_gain_q16, 16, probe_shift) > + WEBRTC_AGC_PROBE_HEADROOM_Q15) + probe_shift--; + probe_val = (int16_t)(1 << probe_shift); + + for (n = 0; n < WEBRTC_AGC_BAND_SAMPLES; n++) + probe[n] = probe_val; + + /* + * cd->gains has WEBRTC_AGC_GAIN_ENVELOPE_LEN (WEBRTC_AGC_SUBFRAMES + 1 = 11) + * elements: gains[0] holds the initial frame gain and gains[1..10] hold + * the end-of-subframe gains for subframes k = 0..9. + */ + gains[0] = ch->inst.gain; + in_ptrs[0] = vad_buf; + in_ptrs[1] = probe; + out_ptrs[0] = vad_buf; + out_ptrs[1] = probe; + + WebRtcAgc_ProcessDigital(&ch->inst, in_ptrs, ARRAY_SIZE(in_ptrs), + out_ptrs, WEBRTC_AGC_BAND_RATE_HZ, 0); + + /* + * Recover the Q16 gain at the start of each 1 ms subframe (10 subframes + * per 10 ms frame, each WEBRTC_AGC_SUBFRAME_BAND_SAMPLES = 16 samples + * long at the 16 kHz band rate). Index WEBRTC_AGC_SUBFRAMES (10) is the + * final element of gains[WEBRTC_AGC_GAIN_ENVELOPE_LEN] (size 11). + */ + for (k = 1; k < WEBRTC_AGC_SUBFRAMES; k++) + gains[k] = Q_SHIFT_LEFT((int32_t)probe[k * WEBRTC_AGC_SUBFRAME_BAND_SAMPLES], + probe_shift, 16); + gains[WEBRTC_AGC_SUBFRAMES] = ch->inst.gain; + + /* + * 2c. Separate full-rate overload limiter peak tracking. + * Measure the true 32/48 kHz peak energy across all samples of each + * 1 ms subframe (k = 0..9, where gains[k + 1] indexes gains[1..10] + * within gains[WEBRTC_AGC_GAIN_ENVELOPE_LEN]) and apply the WebRTC + * overload limiter (-0.1 dB step-down factor 253/256 and 1 ms + * lookahead) so high-frequency peaks attenuated in vad_buf cannot + * clip the full-rate output. + */ + for (k = 0; k < WEBRTC_AGC_SUBFRAMES; k++) { + size_t subfr_samples = WEBRTC_AGC_SUBFRAME_BAND_SAMPLES * num_bands; + const int16_t *subfr = &ch->in_fifo[k * subfr_samples]; + int32_t max_nrg = 0; + int16_t zeros = WEBRTC_AGC_LIMITER_SHIFT_DEFAULT; + int32_t gain32; + + for (n = 0; n < subfr_samples; n++) { + int32_t s = subfr[n]; + int32_t nrg = s * s; + + if (nrg > max_nrg) + max_nrg = nrg; + } + + /* k < WEBRTC_AGC_SUBFRAMES (10), so k + 1 <= 10 < WEBRTC_AGC_GAIN_ENVELOPE_LEN */ + if (gains[k + 1] > WEBRTC_AGC_GAIN_NORM_TLD_Q16) + zeros = 16 - WebRtcSpl_NormW32(gains[k + 1]); + + gain32 = (gains[k + 1] >> zeros) + 1; + gain32 *= gain32; + + while (AGC_MUL32(Q_SHIFT(max_nrg, 30, 18) + 1, gain32) > + WEBRTC_SPL_SHIFT_W32((int32_t)INT16_MAX, + 2 * (1 - zeros + WEBRTC_AGC_LIMITER_SHIFT_DEFAULT))) { + gains[k + 1] = (int32_t)Q_MULTS_32X32((int64_t)gains[k + 1], + WEBRTC_AGC_LIMITER_DECAY_Q8, + 16, 8, 16); + gain32 = (gains[k + 1] >> zeros) + 1; + gain32 *= gain32; + } + } + + for (k = 1; k < WEBRTC_AGC_SUBFRAMES; k++) { + if (gains[k] > gains[k + 1]) + gains[k] = gains[k + 1]; + } + ch->inst.gain = gains[WEBRTC_AGC_SUBFRAMES]; + + /* + * 2d. Apply the 10-subframe interpolated gain trajectory across all + * full-rate 32/48 kHz samples into dst using 64-bit Q-format + * multiplication before saturating to int16_t. + */ + for (k = 0; k < WEBRTC_AGC_SUBFRAMES; k++) { + int32_t delta = gains[k + 1] - gains[k]; + int32_t gain32 = Q_SHIFT_LEFT(gains[k], 16, 20); + + for (n = 0; n < WEBRTC_AGC_SUBFRAME_BAND_SAMPLES; n++) { + int32_t g = Q_SHIFT(gain32, 20, 16); + size_t base = (k * WEBRTC_AGC_SUBFRAME_BAND_SAMPLES + n) * num_bands; + + for (b = 0; b < num_bands; b++) { + int64_t tmp64 = Q_MULTS_32X32((int64_t)ch->in_fifo[base + b], + (int64_t)g, 15, 16, 15); + + dst[base + b] = sat_int16(sat_int32(tmp64)); + } + gain32 += delta; + } + } +} + static int webrtc_agc_real_process(struct processing_module *mod, const void *src, void *dst, int frames, enum sof_ipc_frame fmt) { struct webrtc_agc_comp_data *cd = module_get_private_data(mod); size_t ch_count = cd->channels; + size_t num_bands = (cd->rate <= WEBRTC_AGC_BAND_RATE_HZ) ? + 1 : (size_t)(cd->rate / WEBRTC_AGC_BAND_RATE_HZ); + size_t offset = 0; size_t c, i; if (frames <= 0) return 0; - /* 1. Pull input audio into planar in_fifo */ - { - size_t frames_to_copy = frames; - if (cd->in_fifo_count + frames_to_copy > WEBRTC_AGC_FIFO_FRAMES) - frames_to_copy = WEBRTC_AGC_FIFO_FRAMES - cd->in_fifo_count; - + /* + * Process in chunks of at most (frame_samples - in_fifo_count). + * Because out_fifo is pre-seeded with one 10 ms frame (frame_samples) + * of zeros in prepare()/reset(), the invariant + * (out_fifo_count - out_fifo_rd) == (frame_samples - in_fifo_count) + * holds at the start of every iteration: + * 1. Drain chunk frames from previously processed out_fifo data. + * 2. Enqueue chunk frames of new input into in_fifo. + * 3. When a full 10 ms frame is accumulated (in_fifo_count == + * frame_samples, which simultaneously exhausts out_fifo), process + * the 10 ms block into out_fifo[0..frame_samples - 1] and reset + * both FIFO cursors. + */ + while (offset < (size_t)frames) { + size_t rem = (size_t)frames - offset; + size_t space = (size_t)cd->frame_samples - cd->in_fifo_count; + size_t chunk = (rem < space) ? rem : space; + + /* 1. Drain chunk of processed frames from out_fifo to dst */ if (fmt == SOF_IPC_FRAME_S16_LE) { - const int16_t *s = src; - for (i = 0; i < frames_to_copy; i++) { + int16_t *d = (int16_t *)dst + offset * ch_count; + + for (i = 0; i < chunk; i++) { for (c = 0; c < ch_count; c++) - cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = s[i * ch_count + c]; + d[i * ch_count + c] = + cd->ch_data[c].out_fifo[cd->out_fifo_rd + i]; } } else { - const int32_t *s = src; - for (i = 0; i < frames_to_copy; i++) { + int32_t *d = (int32_t *)dst + offset * ch_count; + + for (i = 0; i < chunk; i++) { for (c = 0; c < ch_count; c++) - cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = (int16_t)(s[i * ch_count + c] >> 16); + d[i * ch_count + c] = + Q_SHIFT_LEFT((int32_t)cd->ch_data[c].out_fifo[cd->out_fifo_rd + i], + 15, 31); } } - cd->in_fifo_count += frames_to_copy; - } - - /* 2. Process complete 10 ms blocks */ - while (cd->in_fifo_count >= cd->frame_samples && - cd->out_fifo_count + cd->frame_samples <= WEBRTC_AGC_FIFO_FRAMES) { - int16_t proc_out[WEBRTC_AGC_FRAME_SAMPLES_MAX]; - - for (c = 0; c < ch_count; c++) { - struct webrtc_agc_channel *ch = &cd->ch_data[c]; - const int16_t *in_ptr = ch->in_fifo; - int16_t *out_ptr = proc_out; - - WebRtcAgc_ProcessDigital(&ch->inst, &in_ptr, 1, &out_ptr, - (cd->rate <= 16000) ? cd->rate : 16000, 0); - - /* Append to out_fifo */ - memcpy(&ch->out_fifo[cd->out_fifo_count], proc_out, - cd->frame_samples * sizeof(int16_t)); + cd->out_fifo_rd += chunk; - /* Shift remaining in_fifo */ - memmove(ch->in_fifo, &ch->in_fifo[cd->frame_samples], - (cd->in_fifo_count - cd->frame_samples) * sizeof(int16_t)); - } - cd->in_fifo_count -= cd->frame_samples; - cd->out_fifo_count += cd->frame_samples; - } + /* 2. Enqueue chunk of input frames into planar in_fifo */ + if (fmt == SOF_IPC_FRAME_S16_LE) { + const int16_t *s = (const int16_t *)src + offset * ch_count; - /* 3. Drain processed samples to dst buffer */ - { - size_t frames_avail = cd->out_fifo_count - cd->out_fifo_rd; - size_t frames_to_write = (frames_avail < (size_t)frames) ? frames_avail : (size_t)frames; - - if (frames_to_write > 0) { - if (fmt == SOF_IPC_FRAME_S16_LE) { - int16_t *d = dst; - for (i = 0; i < frames_to_write; i++) { - for (c = 0; c < ch_count; c++) - d[i * ch_count + c] = cd->ch_data[c].out_fifo[cd->out_fifo_rd + i]; - } - } else { - int32_t *d = dst; - for (i = 0; i < frames_to_write; i++) { - for (c = 0; c < ch_count; c++) - d[i * ch_count + c] = ((int32_t)cd->ch_data[c].out_fifo[cd->out_fifo_rd + i]) << 16; - } + for (i = 0; i < chunk; i++) { + for (c = 0; c < ch_count; c++) + cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = + s[i * ch_count + c]; } + } else { + const int32_t *s = (const int32_t *)src + offset * ch_count; - cd->out_fifo_rd += frames_to_write; - if (cd->out_fifo_rd == cd->out_fifo_count) { - cd->out_fifo_rd = 0; - cd->out_fifo_count = 0; - } else if (cd->out_fifo_rd > 0) { - size_t rem_fifo = cd->out_fifo_count - cd->out_fifo_rd; - - for (c = 0; c < ch_count; c++) { - memmove(cd->ch_data[c].out_fifo, - &cd->ch_data[c].out_fifo[cd->out_fifo_rd], - rem_fifo * sizeof(int16_t)); - } - cd->out_fifo_count = rem_fifo; - cd->out_fifo_rd = 0; + for (i = 0; i < chunk; i++) { + for (c = 0; c < ch_count; c++) + cd->ch_data[c].in_fifo[cd->in_fifo_count + i] = + sat_int16(Q_SHIFT_RND(s[i * ch_count + c], + 31, 15)); } } - - /* If out_fifo had fewer frames than requested, zero-fill remainder */ - if (frames_to_write < (size_t)frames) { - size_t rem = (size_t)frames - frames_to_write; - - if (fmt == SOF_IPC_FRAME_S16_LE) { - int16_t *d = (int16_t *)dst + frames_to_write * ch_count; - - memset(d, 0, rem * ch_count * sizeof(int16_t)); - } else { - int32_t *d = (int32_t *)dst + frames_to_write * ch_count; - - memset(d, 0, rem * ch_count * sizeof(int32_t)); + cd->in_fifo_count += chunk; + + /* 3. Process a complete 10 ms block when in_fifo is full */ + if (cd->in_fifo_count == (size_t)cd->frame_samples) { + for (c = 0; c < ch_count; c++) { + struct webrtc_agc_channel *ch = &cd->ch_data[c]; + int16_t *out_ptr = ch->out_fifo; + + if (num_bands == 1) { + const int16_t *in_ptr = ch->in_fifo; + + WebRtcAgc_ProcessDigital(&ch->inst, &in_ptr, 1, + &out_ptr, cd->rate, 0); + } else { + webrtc_agc_process_multiband(cd, ch, + num_bands, out_ptr); + } } + cd->in_fifo_count = 0; + cd->out_fifo_rd = 0; + cd->out_fifo_count = cd->frame_samples; } + + offset += chunk; } return 0; @@ -172,6 +329,9 @@ static int webrtc_agc_real_reset(struct processing_module *mod) for (c = 0; c < cd->channels; c++) { memset(cd->ch_data[c].in_fifo, 0, sizeof(cd->ch_data[c].in_fifo)); memset(cd->ch_data[c].out_fifo, 0, sizeof(cd->ch_data[c].out_fifo)); + memset(cd->ch_data[c].down_state_32_to_16, 0, + sizeof(cd->ch_data[c].down_state_32_to_16)); + WebRtcSpl_ResetResample48khzTo16khz(&cd->ch_data[c].state_48_to_16); WebRtcAgc_InitDigital(&cd->ch_data[c].inst, cd->mode); WebRtcAgc_CalculateGainTable(cd->ch_data[c].inst.gainTable, cd->compression_gain_db, diff --git a/src/audio/webrtc_agc/webrtc_agc.c b/src/audio/webrtc_agc/webrtc_agc.c index 244d6b7a5a40..7576962196c8 100644 --- a/src/audio/webrtc_agc/webrtc_agc.c +++ b/src/audio/webrtc_agc/webrtc_agc.c @@ -76,8 +76,9 @@ __cold static int webrtc_agc_prepare(struct processing_module *mod, cd->rate = source_get_rate(sources[0]); cd->channels = source_get_channels(sources[0]); - if (cd->rate != 8000 && cd->rate != 16000) { - comp_err(dev, "webrtc_agc: unsupported rate %d (only 8000 and 16000 Hz supported)", + if (cd->rate != 8000 && cd->rate != 16000 && + cd->rate != 32000 && cd->rate != 48000) { + comp_err(dev, "webrtc_agc: unsupported rate %d (only 8/16/32/48 kHz supported)", cd->rate); return -EINVAL; } diff --git a/src/audio/webrtc_agc/webrtc_agc.h b/src/audio/webrtc_agc/webrtc_agc.h index 210aefea1154..b647d5f262ea 100644 --- a/src/audio/webrtc_agc/webrtc_agc.h +++ b/src/audio/webrtc_agc/webrtc_agc.h @@ -16,9 +16,26 @@ typedef struct { uint32_t dummy[64]; } DigitalAgc; #endif -#define WEBRTC_AGC_CHANNELS_MAX 4 -#define WEBRTC_AGC_FRAME_SAMPLES_MAX 480 /* 10 ms @ 48 kHz */ -#define WEBRTC_AGC_FIFO_FRAMES (WEBRTC_AGC_FRAME_SAMPLES_MAX * 2) +#define WEBRTC_AGC_CHANNELS_MAX 4 +/* + * WebRTC's WebRtcAgc_ProcessDigital() algorithm operates on a fixed 10 ms frame + * (10 x 1 ms subframes); the module's internal framing FIFO decouples this from + * the SOF pipeline period. + */ +#define WEBRTC_AGC_FRAME_MS 10 +/* 10 x 1 ms subframes per 10 ms frame; 11 envelope boundary points (0..10) */ +#define WEBRTC_AGC_SUBFRAMES 10 +#define WEBRTC_AGC_GAIN_ENVELOPE_LEN (WEBRTC_AGC_SUBFRAMES + 1) +#define WEBRTC_AGC_BAND_RATE_HZ 16000 +#define WEBRTC_AGC_MAX_RATE_HZ 48000 +#define WEBRTC_AGC_BAND_SAMPLES \ + ((WEBRTC_AGC_BAND_RATE_HZ * WEBRTC_AGC_FRAME_MS) / 1000) +#define WEBRTC_AGC_SUBFRAME_BAND_SAMPLES \ + (WEBRTC_AGC_BAND_SAMPLES / WEBRTC_AGC_SUBFRAMES) +#define WEBRTC_AGC_FRAME_SAMPLES_MAX \ + ((WEBRTC_AGC_MAX_RATE_HZ * WEBRTC_AGC_FRAME_MS) / 1000) +#define WEBRTC_AGC_FIFO_FRAMES WEBRTC_AGC_FRAME_SAMPLES_MAX +#define WEBRTC_AGC_RESAMP_TMPMEM_LEN (WEBRTC_AGC_FRAME_SAMPLES_MAX + 16) struct webrtc_agc_backend { const char *name; @@ -33,6 +50,10 @@ struct webrtc_agc_backend { struct webrtc_agc_channel { DigitalAgc inst; +#if !CONFIG_COMP_WEBRTC_AGC_STUB + int32_t down_state_32_to_16[8]; + WebRtcSpl_State48khzTo16khz state_48_to_16; +#endif int16_t in_fifo[WEBRTC_AGC_FIFO_FRAMES]; int16_t out_fifo[WEBRTC_AGC_FIFO_FRAMES]; }; @@ -41,6 +62,14 @@ struct webrtc_agc_comp_data { struct webrtc_agc_channel ch_data[WEBRTC_AGC_CHANNELS_MAX]; const struct webrtc_agc_backend *backend; +#if !CONFIG_COMP_WEBRTC_AGC_STUB + /* Scratch buffers for multi-band (32/48 kHz) processing */ + int16_t vad_buf[WEBRTC_AGC_BAND_SAMPLES]; + int16_t probe[WEBRTC_AGC_BAND_SAMPLES]; + int32_t gains[WEBRTC_AGC_GAIN_ENVELOPE_LEN]; + int32_t resamp_tmpmem[WEBRTC_AGC_RESAMP_TMPMEM_LEN]; +#endif + int channels; int rate; int frame_samples; /* samples per 10 ms frame */ diff --git a/src/audio/webrtc_hpf/Kconfig b/src/audio/webrtc_hpf/Kconfig index 07a5623984ed..6d6d57a2e728 100644 --- a/src/audio/webrtc_hpf/Kconfig +++ b/src/audio/webrtc_hpf/Kconfig @@ -11,7 +11,7 @@ if COMP_WEBRTC_HPF config COMP_WEBRTC_HPF_STUB bool "Build with pass-through stub backend" - default n + default y if COMP_STUBS help Select to build with the pass-through stub backend instead of the real filter. diff --git a/tools/topology/topology2/include/components/webrtc-aec.conf b/tools/topology/topology2/include/components/webrtc-aec.conf index f83f8cd5d25e..f90cd3d8f108 100644 --- a/tools/topology/topology2/include/components/webrtc-aec.conf +++ b/tools/topology/topology2/include/components/webrtc-aec.conf @@ -61,6 +61,7 @@ Class.Widget."webrtc-aec" { !immutable [ "uuid" "type" + "scheduler_domain" ] !deprecated [ @@ -135,10 +136,12 @@ Class.Widget."webrtc-aec" { ] # UUID: e5da7b5b-133a-ba46-b517651d6300bb83 - uuid "5b:7b:da:e5:3a:13:46:ba:b5:17:65:1d:63:00:bb:83" - type "effect" - no_pm "true" - cpc 2000 + uuid "5b:7b:da:e5:3a:13:46:ba:b5:17:65:1d:63:00:bb:83" + type "effect" + no_pm "true" + cpc 2000 + scheduler_domain "DP" + stack_bytes_requirement 16384 num_input_pins 2 num_output_pins 1 } diff --git a/tools/topology/topology2/include/components/webrtc-agc.conf b/tools/topology/topology2/include/components/webrtc-agc.conf index e0a2e9b684d5..9cc0ce4cf6ef 100644 --- a/tools/topology/topology2/include/components/webrtc-agc.conf +++ b/tools/topology/topology2/include/components/webrtc-agc.conf @@ -3,7 +3,7 @@ # # WebRTC Digital Automatic Gain Control widget (fixed-point). # Single input, single output. Dynamically normalizes speech levels (+9 dB boost, limiter). -# Supports 8 kHz and 16 kHz, S16 and S32 formats. +# Supports 8, 16, 32, and 48 kHz, S16 and S32 formats. # # Usage: # Object.Widget.webrtc-agc."N" { @@ -70,10 +70,34 @@ Class.Widget."webrtc-agc" { } } - num_input_audio_formats 4 - num_output_audio_formats 4 + num_input_audio_formats 8 + num_output_audio_formats 8 Object.Base.input_audio_format [ + # S16 stereo 48 kHz + { + in_rate 48000 + in_bit_depth 16 + in_valid_bit_depth 16 + } + # S32 stereo 48 kHz + { + in_rate 48000 + in_bit_depth 32 + in_valid_bit_depth 32 + } + # S16 stereo 32 kHz + { + in_rate 32000 + in_bit_depth 16 + in_valid_bit_depth 16 + } + # S32 stereo 32 kHz + { + in_rate 32000 + in_bit_depth 32 + in_valid_bit_depth 32 + } # S16 stereo 16 kHz { in_rate 16000 @@ -101,6 +125,26 @@ Class.Widget."webrtc-agc" { ] Object.Base.output_audio_format [ + { + out_rate 48000 + out_bit_depth 16 + out_valid_bit_depth 16 + } + { + out_rate 48000 + out_bit_depth 32 + out_valid_bit_depth 32 + } + { + out_rate 32000 + out_bit_depth 16 + out_valid_bit_depth 16 + } + { + out_rate 32000 + out_bit_depth 32 + out_valid_bit_depth 32 + } { out_rate 16000 out_bit_depth 16