From 88fd203d3b842f7db2cda34b72452a1ced711ed3 Mon Sep 17 00:00:00 2001 From: Ptits de Barbe Date: Sun, 26 Aug 2018 11:45:28 +0300 Subject: [PATCH] AEC filter from Speex. --- configure | 4 + libavfilter/Makefile | 1 + libavfilter/af_libspeex_aec.c | 293 ++++++++++++++++++++++++++++++++++ libavfilter/allfilters.c | 1 + 4 files changed, 299 insertions(+) create mode 100644 libavfilter/af_libspeex_aec.c diff --git a/configure b/configure index e718c1531c..45c8cba724 100755 --- a/configure +++ b/configure @@ -259,6 +259,7 @@ External library support: --enable-libsnappy enable Snappy compression, needed for hap encoding [no] --enable-libsoxr enable Include libsoxr resampling [no] --enable-libspeex enable Speex de/encoding via libspeex [no] + --enable-libspeexdsp enable Speex AEC [no] --enable-libsrt enable Haivision SRT protocol via libsrt [no] --enable-libssh enable SFTP protocol via libssh [no] --enable-libtensorflow enable TensorFlow as a DNN module backend @@ -1717,6 +1718,7 @@ EXTERNAL_LIBRARY_LIST=" libsnappy libsoxr libspeex + libspeexdsp libsrt libssh libtensorflow @@ -3086,6 +3088,7 @@ libopus_encoder_select="audio_frame_queue" librsvg_decoder_deps="librsvg" libshine_encoder_deps="libshine" libshine_encoder_select="audio_frame_queue" +libspeex_aec_filter_deps="libspeexdsp" libspeex_decoder_deps="libspeex" libspeex_encoder_deps="libspeex" libspeex_encoder_select="audio_frame_queue" @@ -6079,6 +6082,7 @@ enabled libsnappy && require libsnappy snappy-c.h snappy_compress -lsnap enabled libsoxr && require libsoxr soxr.h soxr_create -lsoxr enabled libssh && require_pkg_config libssh libssh libssh/sftp.h sftp_init enabled libspeex && require_pkg_config libspeex speex speex/speex.h speex_decoder_init +enabled libspeexdsp && require_pkg_config libspeexdsp speexdsp speex/speex_echo.h speex_echo_state_init_mc enabled libsrt && require_pkg_config libsrt "srt >= 1.2.0" srt/srt.h srt_socket enabled libtensorflow && require libtensorflow tensorflow/c/c_api.h TF_Version -ltensorflow enabled libtesseract && require_pkg_config libtesseract tesseract tesseract/capi.h TessBaseAPICreate diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 245302bbe8..1c5502f9a6 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -108,6 +108,7 @@ OBJS-$(CONFIG_HIGHPASS_FILTER) += af_biquads.o OBJS-$(CONFIG_HIGHSHELF_FILTER) += af_biquads.o OBJS-$(CONFIG_JOIN_FILTER) += af_join.o OBJS-$(CONFIG_LADSPA_FILTER) += af_ladspa.o +OBJS-$(CONFIG_LIBSPEEX_AEC_FILTER) += af_libspeex_aec.o OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o ebur128.o OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o OBJS-$(CONFIG_LOWSHELF_FILTER) += af_biquads.o diff --git a/libavfilter/af_libspeex_aec.c b/libavfilter/af_libspeex_aec.c new file mode 100644 index 0000000000..eb96f4e533 --- /dev/null +++ b/libavfilter/af_libspeex_aec.c @@ -0,0 +1,293 @@ +/* + * Speex-based Acoustic Echo Canceller. + * Copyright (c) 2018 Arseniy Skvortsov + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Speex-based Acoustic Echo Canceller + * + * Tries to remove the second input from the first input + * as if the 1st was recorded during playback of the 2nd. + * No delay is assumed. + */ + +#include + +#include "libavutil/attributes.h" +#include "libavutil/opt.h" +#include "libavutil/samplefmt.h" + +#include "audio.h" +#include "avfilter.h" +#include "filters.h" +#include "formats.h" +#include "internal.h" + +typedef struct AECContext { + const AVClass *class; /**< class for AVOptions */ + + int frame_size; /**< number of samples */ + int filter_length; /**< number of samples to search echo in */ + int nb_mics; /**< number of channels in recording input */ + int nb_speakers; /**< number of channels in playback input */ + + SpeexEchoState *state; /**< Speex's own state */ +} AECContext; + +#define OFFSET(x) offsetof(AECContext, x) +#define A AV_OPT_FLAG_AUDIO_PARAM +#define F AV_OPT_FLAG_FILTERING_PARAM +static const AVOption libspeex_aec_options[] = { + { "frame_size", "Number of samples to process at one time (should correspond to 20 ms, preferably 2ⁿ for FFT)", + OFFSET(frame_size), AV_OPT_TYPE_INT, { .i64 = 1024 }, 1, INT_MAX, A|F }, + { "filter_length", "Number of samples of echo to cancel " + "(should generally correspond to 100-500 ms or a ⅓ of room reverberation time)", + OFFSET(filter_length), AV_OPT_TYPE_INT, { .i64 = 5000 }, 1, INT_MAX, A|F }, + { NULL } +}; + +AVFILTER_DEFINE_CLASS(libspeex_aec); + +static int query_formats(AVFilterContext *ctx) +{ + AVFilterFormats *formats = NULL; + AVFilterChannelLayouts *mics_layouts = NULL; + //Filter supports a different layout for this input, need to create another set to enable separate negotiation. + AVFilterChannelLayouts *speakers_layouts = NULL; + AVFilterFormats *sample_rates = NULL; + int ret; + + if ((ret = ff_add_format(&formats, AV_SAMPLE_FMT_S16P)) < 0) { + return ret; + } + + mics_layouts = ff_all_channel_counts(); + if (!mics_layouts) { + ret = AVERROR(ENOMEM); + goto clean; + } + + speakers_layouts = ff_all_channel_counts(); + if (!speakers_layouts) { + ret = AVERROR(ENOMEM); + goto clean; + } + + sample_rates = ff_all_samplerates(); + if (!sample_rates) { + ret = AVERROR(ENOMEM); + goto clean; + } + + ret = ff_channel_layouts_ref(speakers_layouts, &ctx->inputs[1]->out_channel_layouts); + if (ret < 0) { + goto clean; + } + + //A fail in the middle of these can leave references to a freed memory. + //True not only for this (caller) function, but for all callees too. + if ((ret = ff_set_common_formats (ctx, formats )) < 0 || + (ret = ff_set_common_channel_layouts(ctx, mics_layouts)) < 0 || + (ret = ff_set_common_samplerates (ctx, sample_rates)) < 0) { + goto clean; + } + return 0; + + clean: + if (sample_rates) { + av_freep(&sample_rates); + } + + if (speakers_layouts) { + av_freep(&speakers_layouts); + } + + if (mics_layouts) { + av_freep(&mics_layouts); + } + + if (formats) { + av_freep(&formats->formats); + av_freep(&formats); + } + return ret; +} + +static int config_state(AVFilterContext *ctx) +{ + AECContext *self = ctx->priv; + if (!self->nb_mics || !self->nb_speakers) { + return 0; + } + self->state = speex_echo_state_init_mc(self->frame_size, self->filter_length, self->nb_mics, self->nb_speakers); + if (self->state <= 0) { + av_log(ctx, AV_LOG_ERROR, "Cannot initialize libspeex AEC state!"); + return AVERROR(ENOMEM); + } + + av_log(ctx, AV_LOG_VERBOSE, "mics:%d speakers:%d\n", self->nb_mics, self->nb_speakers); + + return 0; +} + +static int config_record_input(AVFilterLink *inlink) +{ + AVFilterContext *ctx = inlink->dst; + AECContext *self = ctx->priv; + self->nb_mics = inlink->channels; + inlink->request_samples = self->frame_size; + return config_state(ctx); +} + +static int config_playback_input(AVFilterLink *inlink) +{ + AVFilterContext *ctx = inlink->dst; + AECContext *self = ctx->priv; + self->nb_speakers = inlink->channels; + inlink->request_samples = self->frame_size; + return config_state(ctx); +} + +static int config_output(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + AVFilterLink *inlink = ctx->inputs[0]; + if (inlink) { + if (!outlink->time_base.num && !outlink->time_base.den) { + outlink->time_base = inlink->time_base; + } + } + return 0; +} + +static int filter_frames(const AVFilterContext *ctx) { + AVFilterLink *outlink = ctx->outputs[0]; + AECContext *self = ctx->priv; + AVFilterLink *input_to_clean = ctx->inputs[0]; + AVFrame *recorded; + AVFrame *played; + AVFrame *cleaned; + int ret; + + if (!ff_inlink_check_available_frame(input_to_clean) || + !ff_inlink_check_available_frame(ctx->inputs[1])) { + return 1; + } + + ret = ff_inlink_consume_frame(input_to_clean, &recorded); + if (ret < 0) { + return ret; + } + ret = ff_inlink_consume_frame(ctx->inputs[1], &played); + if (ret < 0) { + av_frame_free(&recorded); + return ret; + } + cleaned = ff_get_audio_buffer(outlink, self->frame_size); + if (!cleaned) { + ret = AVERROR(ENOMEM); + av_frame_free(&recorded); + av_frame_free(&played); + return ret; + } + speex_echo_cancellation( + self->state, + (const spx_int16_t *) *recorded->extended_data, + (const spx_int16_t *) *played->extended_data, + (spx_int16_t *) *cleaned->extended_data + ); + + av_frame_free(&recorded); + av_frame_free(&played); + return ff_filter_frame(outlink, cleaned); +} + +static int activate(AVFilterContext *ctx) +{ + AVFilterLink *outlink = ctx->outputs[0]; + int ret; + + FF_FILTER_FORWARD_STATUS_BACK_ALL(outlink, ctx); + + ret = filter_frames(ctx); + if (ret <= 0) { + return ret; + } + + FF_FILTER_FORWARD_STATUS(ctx->inputs[0], outlink); + FF_FILTER_FORWARD_STATUS(ctx->inputs[1], outlink); + if (ff_outlink_frame_wanted(outlink)) { + //One is definitely missing a frame, because we did no processing. + if (!ff_inlink_check_available_frame(ctx->inputs[0])) { + ff_inlink_request_frame(ctx->inputs[0]); + } + if (!ff_inlink_check_available_frame(ctx->inputs[1])) { + ff_inlink_request_frame(ctx->inputs[1]); + } + return 0; + } + return FFERROR_NOT_READY; +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + AECContext *self = ctx->priv; + + if (self->state) { + speex_echo_state_destroy(self->state); + } +} + +static const AVFilterPad avfilter_af_libspeex_aec_inputs[] = { + { + .name = "record", + .type = AVMEDIA_TYPE_AUDIO, + .needs_fifo = 1, + .config_props = config_record_input, + }, + { + .name = "playback", + .type = AVMEDIA_TYPE_AUDIO, + .needs_fifo = 1, + .config_props = config_playback_input, + }, + { NULL } +}; + +static const AVFilterPad avfilter_af_libspeex_aec_outputs[] = { + { + .name = "cleaned", + .type = AVMEDIA_TYPE_AUDIO, + .config_props = config_output, + }, + { NULL } +}; + +AVFilter ff_af_libspeex_aec = { + .name = "libspeex_aec", + .description = NULL_IF_CONFIG_SMALL("Speex-based acoustic echo cancellation (AEC)."), + .priv_size = sizeof(AECContext), + .priv_class = &libspeex_aec_class, + .uninit = uninit, + .activate = activate, + .query_formats = query_formats, + .inputs = avfilter_af_libspeex_aec_inputs, + .outputs = avfilter_af_libspeex_aec_outputs, +}; diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 2d19929bdc..3db1436c82 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -101,6 +101,7 @@ extern AVFilter ff_af_highpass; extern AVFilter ff_af_highshelf; extern AVFilter ff_af_join; extern AVFilter ff_af_ladspa; +extern AVFilter ff_af_libspeex_aec; extern AVFilter ff_af_loudnorm; extern AVFilter ff_af_lowpass; extern AVFilter ff_af_lowshelf; -- 2.18.0