From: Arun Raghavan <git@xxxxxxxxxxxxxxxx>
This is required to have unequal channel counts on capture in and out
streams, which is needed for beamforming to work. The deinterleaved API
only works with floating point samples.
---
src/modules/echo-cancel/echo-cancel.h | 1 +
src/modules/echo-cancel/module-echo-cancel.c | 4 +-
src/modules/echo-cancel/webrtc.cc | 56 ++++++++++++++++------------
3 files changed, 36 insertions(+), 25 deletions(-)
diff --git a/src/modules/echo-cancel/echo-cancel.h b/src/modules/echo-cancel/echo-cancel.h
index ab0c5e9..83f8b85 100644
--- a/src/modules/echo-cancel/echo-cancel.h
+++ b/src/modules/echo-cancel/echo-cancel.h
@@ -66,6 +66,7 @@ struct pa_echo_canceller_params {
void *apm;
unsigned int blocksize; /* in frames */
pa_sample_spec rec_ss, play_ss, out_ss;
+ float *rec_buffer[PA_CHANNELS_MAX], *play_buffer[PA_CHANNELS_MAX]; /* for deinterleaved buffers */
void *trace_callback;
bool agc;
bool first;
diff --git a/src/modules/echo-cancel/module-echo-cancel.c b/src/modules/echo-cancel/module-echo-cancel.c
index 484281f..4e898d0 100644
--- a/src/modules/echo-cancel/module-echo-cancel.c
+++ b/src/modules/echo-cancel/module-echo-cancel.c
@@ -2158,12 +2158,12 @@ int main(int argc, char* argv[]) {
goto fail;
}
- source_ss.format = PA_SAMPLE_S16LE;
+ source_ss.format = PA_SAMPLE_FLOAT32LE;
source_ss.rate = DEFAULT_RATE;
source_ss.channels = DEFAULT_CHANNELS;
pa_channel_map_init_auto(&source_map, source_ss.channels, PA_CHANNEL_MAP_DEFAULT);
- sink_ss.format = PA_SAMPLE_S16LE;
+ sink_ss.format = PA_SAMPLE_FLOAT32LE;
sink_ss.rate = DEFAULT_RATE;
sink_ss.channels = DEFAULT_CHANNELS;
pa_channel_map_init_auto(&sink_map, sink_ss.channels, PA_CHANNEL_MAP_DEFAULT);
diff --git a/src/modules/echo-cancel/webrtc.cc b/src/modules/echo-cancel/webrtc.cc
index 35ba93f..385c211 100644
--- a/src/modules/echo-cancel/webrtc.cc
+++ b/src/modules/echo-cancel/webrtc.cc
@@ -117,8 +117,8 @@ static void pa_webrtc_ec_fixate_spec(pa_sample_spec *rec_ss, pa_channel_map *rec
pa_sample_spec *play_ss, pa_channel_map *play_map,
pa_sample_spec *out_ss, pa_channel_map *out_map)
{
- rec_ss->format = PA_SAMPLE_S16NE;
- play_ss->format = PA_SAMPLE_S16NE;
+ rec_ss->format = PA_SAMPLE_FLOAT32NE;
+ play_ss->format = PA_SAMPLE_FLOAT32NE;
/* AudioProcessing expects one of the following rates */
if (rec_ss->rate >= 48000)
@@ -147,7 +147,7 @@ bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
webrtc::ProcessingConfig pconfig;
webrtc::Config config;
bool hpf, ns, agc, dgc, mobile, cn, vad, ext_filter, intelligibility, experimental_agc;
- int rm = -1;
+ int rm = -1, i;
uint32_t agc_start_volume;
pa_modargs *ma;
bool trace = false;
@@ -340,6 +340,11 @@ bool pa_webrtc_ec_init(pa_core *c, pa_echo_canceller *ec,
*nframes = ec->params.webrtc.blocksize;
ec->params.webrtc.first = true;
+ for (i = 0; i < rec_ss->channels; i++)
+ ec->params.webrtc.rec_buffer[i] = pa_xnew(float, *nframes);
+ for (i = 0; i < play_ss->channels; i++)
+ ec->params.webrtc.play_buffer[i] = pa_xnew(float, *nframes);
+
pa_modargs_free(ma);
return true;
@@ -357,36 +362,34 @@ fail:
void pa_webrtc_ec_play(pa_echo_canceller *ec, const uint8_t *play) {
webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
- webrtc::AudioFrame play_frame;
const pa_sample_spec *ss = &ec->params.webrtc.play_ss;
+ int n = ec->params.webrtc.blocksize;
+ float **buf = ec->params.webrtc.play_buffer;
+ webrtc::StreamConfig config(ss->rate, ss->channels, false);
- play_frame.num_channels_ = ss->channels;
- play_frame.sample_rate_hz_ = ss->rate;
- play_frame.interleaved_ = true;
- play_frame.samples_per_channel_ = ec->params.webrtc.blocksize;
-
- pa_assert(play_frame.samples_per_channel_ <= webrtc::AudioFrame::kMaxDataSizeSamples);
- memcpy(play_frame.data_, play, ec->params.webrtc.blocksize * pa_frame_size(ss));
+ pa_deinterleave(play, (void **) buf, ss->channels, pa_sample_size(ss), n);
- if (apm->ProcessReverseStream(&play_frame) != webrtc::AudioProcessing::kNoError)
+ if (apm->ProcessReverseStream(buf, config, config, buf) != webrtc::AudioProcessing::kNoError)
pa_log("Failed to process playback stream");
+
+ /* FIXME: we need to be able to modify playback samples, which we can't
+ * currently do. This is because module-echo-cancel processes playback
+ * frames in the source thread, and just stores playback chunks as they
+ * pass through the sink. */
}
void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out) {
webrtc::AudioProcessing *apm = (webrtc::AudioProcessing*)ec->params.webrtc.apm;
- webrtc::AudioFrame out_frame;
const pa_sample_spec *rec_ss = &ec->params.webrtc.rec_ss;
const pa_sample_spec *out_ss = &ec->params.webrtc.out_ss;
+ float **buf = ec->params.webrtc.rec_buffer;
+ int n = ec->params.webrtc.blocksize;
pa_cvolume v;
int old_volume, new_volume;
+ webrtc::StreamConfig rec_config(rec_ss->rate, rec_ss->channels, false);
+ webrtc::StreamConfig out_config(out_ss->rate, out_ss->channels, false);
- out_frame.num_channels_ = rec_ss->channels;
- out_frame.sample_rate_hz_ = rec_ss->rate;
- out_frame.interleaved_ = true;
- out_frame.samples_per_channel_ = ec->params.webrtc.blocksize;
-
- pa_assert(out_frame.samples_per_channel_ <= webrtc::AudioFrame::kMaxDataSizeSamples);
- memcpy(out_frame.data_, rec, ec->params.webrtc.blocksize * pa_frame_size(rec_ss));
+ pa_deinterleave(rec, (void **) buf, rec_ss->channels, pa_sample_size(rec_ss), n);
if (ec->params.webrtc.agc) {
pa_cvolume_init(&v);
@@ -396,7 +399,7 @@ void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out
}
apm->set_stream_delay_ms(0);
- pa_assert_se(apm->ProcessStream(&out_frame) == webrtc::AudioProcessing::kNoError);
+ pa_assert_se(apm->ProcessStream(buf, rec_config, out_config, buf) == webrtc::AudioProcessing::kNoError);
if (ec->params.webrtc.agc) {
if (PA_UNLIKELY(ec->params.webrtc.first)) {
@@ -411,12 +414,12 @@ void pa_webrtc_ec_record(pa_echo_canceller *ec, const uint8_t *rec, uint8_t *out
}
if (old_volume != new_volume) {
- pa_cvolume_set(&v, rec_ss->channels, webrtc_volume_to_pa(new_volume));
+ pa_cvolume_set(&v, out_ss->channels, webrtc_volume_to_pa(new_volume));
pa_echo_canceller_set_capture_volume(ec, &v);
}
}
- memcpy(out, out_frame.data_, ec->params.webrtc.blocksize * pa_frame_size(out_ss));
+ pa_interleave((const void **) buf, out_ss->channels, out, pa_sample_size(out_ss), n);
}
void pa_webrtc_ec_set_drift(pa_echo_canceller *ec, float drift) {
@@ -431,6 +434,8 @@ void pa_webrtc_ec_run(pa_echo_canceller *ec, const uint8_t *rec, const uint8_t *
}
void pa_webrtc_ec_done(pa_echo_canceller *ec) {
+ int i;
+
if (ec->params.webrtc.trace_callback) {
webrtc::Trace::ReturnTrace();
delete ((PaWebrtcTraceCallback *) ec->params.webrtc.trace_callback);
@@ -440,4 +445,9 @@ void pa_webrtc_ec_done(pa_echo_canceller *ec) {
delete (webrtc::AudioProcessing*)ec->params.webrtc.apm;
ec->params.webrtc.apm = NULL;
}
+
+ for (i = 0; i < ec->params.webrtc.rec_ss.channels; i++)
+ pa_xfree(ec->params.webrtc.rec_buffer[i]);
+ for (i = 0; i < ec->params.webrtc.play_ss.channels; i++)
+ pa_xfree(ec->params.webrtc.play_buffer[i]);
}
--
2.5.0
