FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2022 Paul B Mahol
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or
7			* modify it under the terms of the GNU Lesser General Public
8			* License as published by the Free Software Foundation; either
9			* version 2.1 of the License, or (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14			* Lesser General Public License for more details.
15			*
16			* You should have received a copy of the GNU Lesser General Public
17			* License along with FFmpeg; if not, write to the Free Software
18			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19			*/
20
21			#include <float.h>
22
23			#include "libavutil/eval.h"
24			#include "libavutil/ffmath.h"
25			#include "libavutil/mem.h"
26			#include "libavutil/opt.h"
27			#include "libavutil/tx.h"
28			#include "audio.h"
29			#include "avfilter.h"
30			#include "filters.h"
31			#include "internal.h"
32
33			static const char * const var_names[] = {
34			"ch", ///< the value of the current channel
35			"sn", ///< number of samples
36			"nb_channels",
37			"t", ///< timestamp expressed in seconds
38			"sr", ///< sample rate
39			"p", ///< input power in dB for frequency bin
40			"f", ///< frequency in Hz
41			NULL
42			};
43
44			enum var_name {
45			VAR_CH,
46			VAR_SN,
47			VAR_NB_CHANNELS,
48			VAR_T,
49			VAR_SR,
50			VAR_P,
51			VAR_F,
52			VAR_VARS_NB
53			};
54
55			typedef struct AudioDRCContext {
56			const AVClass *class;
57
58			double attack_ms;
59			double release_ms;
60			char *expr_str;
61
62			double attack;
63			double release;
64
65			int fft_size;
66			int overlap;
67			int channels;
68
69			float fx;
70			float *window;
71
72			AVFrame *drc_frame;
73			AVFrame *energy;
74			AVFrame *envelope;
75			AVFrame *factors;
76			AVFrame *in;
77			AVFrame *in_buffer;
78			AVFrame *in_frame;
79			AVFrame *out_dist_frame;
80			AVFrame *spectrum_buf;
81			AVFrame *target_gain;
82			AVFrame *windowed_frame;
83
84			char *channels_to_filter;
85			AVChannelLayout ch_layout;
86
87			AVTXContext **tx_ctx;
88			av_tx_fn tx_fn;
89			AVTXContext **itx_ctx;
90			av_tx_fn itx_fn;
91
92			AVExpr *expr;
93			double var_values[VAR_VARS_NB];
94			} AudioDRCContext;
95
96			#define OFFSET(x) offsetof(AudioDRCContext, x)
97			#define FLAGS AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_RUNTIME_PARAM
98
99			static const AVOption adrc_options[] = {
100			{ "transfer", "set the transfer expression", OFFSET(expr_str), AV_OPT_TYPE_STRING, {.str="p"}, 0, 0, FLAGS },
101			{ "attack", "set the attack", OFFSET(attack_ms), AV_OPT_TYPE_DOUBLE, {.dbl=50.}, 1, 1000, FLAGS },
102			{ "release", "set the release", OFFSET(release_ms), AV_OPT_TYPE_DOUBLE, {.dbl=100.}, 5, 2000, FLAGS },
103			{ "channels", "set channels to filter",OFFSET(channels_to_filter),AV_OPT_TYPE_STRING,{.str="all"},0, 0, FLAGS },
104			{NULL}
105			};
106
107			AVFILTER_DEFINE_CLASS(adrc);
108
109		✗	static void generate_hann_window(float *window, int size)
110			{
111		✗	for (int i = 0; i < size; i++) {
112		✗	float value = 0.5f * (1.f - cosf(2.f * M_PI * i / size));
113
114		✗	window[i] = value;
115			}
116		✗	}
117
118		✗	static int config_input(AVFilterLink *inlink)
119			{
120		✗	AVFilterContext *ctx = inlink->dst;
121		✗	AudioDRCContext *s = ctx->priv;
122			float scale;
123			int ret;
124
125		✗	s->fft_size = inlink->sample_rate > 100000 ? 1024 : inlink->sample_rate > 50000 ? 512 : 256;
126		✗	s->fx = inlink->sample_rate * 0.5f / (s->fft_size / 2 + 1);
127		✗	s->overlap = s->fft_size / 4;
128
129		✗	s->window = av_calloc(s->fft_size, sizeof(*s->window));
130		✗	if (!s->window)
131		✗	return AVERROR(ENOMEM);
132
133		✗	s->drc_frame = ff_get_audio_buffer(inlink, s->fft_size * 2);
134		✗	s->energy = ff_get_audio_buffer(inlink, s->fft_size / 2 + 1);
135		✗	s->envelope = ff_get_audio_buffer(inlink, s->fft_size / 2 + 1);
136		✗	s->factors = ff_get_audio_buffer(inlink, s->fft_size / 2 + 1);
137		✗	s->in_buffer = ff_get_audio_buffer(inlink, s->fft_size * 2);
138		✗	s->in_frame = ff_get_audio_buffer(inlink, s->fft_size * 2);
139		✗	s->out_dist_frame = ff_get_audio_buffer(inlink, s->fft_size * 2);
140		✗	s->spectrum_buf = ff_get_audio_buffer(inlink, s->fft_size * 2);
141		✗	s->target_gain = ff_get_audio_buffer(inlink, s->fft_size / 2 + 1);
142		✗	s->windowed_frame = ff_get_audio_buffer(inlink, s->fft_size * 2);
143		✗	if (!s->in_buffer || !s->in_frame || !s->target_gain ||
144		✗	!s->out_dist_frame || !s->windowed_frame || !s->envelope ||
145		✗	!s->drc_frame || !s->spectrum_buf || !s->energy || !s->factors)
146		✗	return AVERROR(ENOMEM);
147
148		✗	generate_hann_window(s->window, s->fft_size);
149
150		✗	s->channels = inlink->ch_layout.nb_channels;
151
152		✗	s->tx_ctx = av_calloc(s->channels, sizeof(*s->tx_ctx));
153		✗	s->itx_ctx = av_calloc(s->channels, sizeof(*s->itx_ctx));
154		✗	if (!s->tx_ctx || !s->itx_ctx)
155		✗	return AVERROR(ENOMEM);
156
157		✗	for (int ch = 0; ch < s->channels; ch++) {
158		✗	scale = 1.f / s->fft_size;
159		✗	ret = av_tx_init(&s->tx_ctx[ch], &s->tx_fn, AV_TX_FLOAT_RDFT, 0, s->fft_size, &scale, 0);
160		✗	if (ret < 0)
161		✗	return ret;
162
163		✗	scale = 1.f;
164		✗	ret = av_tx_init(&s->itx_ctx[ch], &s->itx_fn, AV_TX_FLOAT_RDFT, 1, s->fft_size, &scale, 0);
165		✗	if (ret < 0)
166		✗	return ret;
167			}
168
169		✗	s->var_values[VAR_SR] = inlink->sample_rate;
170		✗	s->var_values[VAR_NB_CHANNELS] = s->channels;
171
172		✗	return av_expr_parse(&s->expr, s->expr_str, var_names, NULL, NULL,
173			NULL, NULL, 0, ctx);
174			}
175
176		✗	static void apply_window(AudioDRCContext *s,
177			const float *in_frame, float *out_frame, const int add_to_out_frame)
178			{
179		✗	const float *window = s->window;
180		✗	const int fft_size = s->fft_size;
181
182		✗	if (add_to_out_frame) {
183		✗	for (int i = 0; i < fft_size; i++)
184		✗	out_frame[i] += in_frame[i] * window[i];
185			} else {
186		✗	for (int i = 0; i < fft_size; i++)
187		✗	out_frame[i] = in_frame[i] * window[i];
188			}
189		✗	}
190
191		✗	static float sqrf(float x)
192			{
193		✗	return x * x;
194			}
195
196		✗	static void get_energy(AVFilterContext *ctx,
197			int len,
198			float *energy,
199			const float *spectral)
200			{
201		✗	for (int n = 0; n < len; n++) {
202		✗	energy[n] = 10.f * log10f(sqrf(spectral[2 * n]) + sqrf(spectral[2 * n + 1]));
203		✗	if (!isnormal(energy[n]))
204		✗	energy[n] = -351.f;
205			}
206		✗	}
207
208		✗	static void get_target_gain(AVFilterContext *ctx,
209			int len,
210			float *gain,
211			const float *energy,
212			double *var_values,
213			float fx, int bypass)
214			{
215		✗	AudioDRCContext *s = ctx->priv;
216
217		✗	if (bypass) {
218		✗	memcpy(gain, energy, sizeof(*gain) * len);
219		✗	return;
220			}
221
222		✗	for (int n = 0; n < len; n++) {
223		✗	const float Xg = energy[n];
224
225		✗	var_values[VAR_P] = Xg;
226		✗	var_values[VAR_F] = n * fx;
227
228		✗	gain[n] = av_expr_eval(s->expr, var_values, s);
229			}
230			}
231
232		✗	static void get_envelope(AVFilterContext *ctx,
233			int len,
234			float *envelope,
235			const float *energy,
236			const float *gain)
237			{
238		✗	AudioDRCContext *s = ctx->priv;
239		✗	const float release = s->release;
240		✗	const float attack = s->attack;
241
242		✗	for (int n = 0; n < len; n++) {
243		✗	const float Bg = gain[n] - energy[n];
244		✗	const float Vg = envelope[n];
245
246		✗	if (Bg > Vg) {
247		✗	envelope[n] = attack * Vg + (1.f - attack) * Bg;
248		✗	} else if (Bg <= Vg) {
249		✗	envelope[n] = release * Vg + (1.f - release) * Bg;
250			} else {
251		✗	envelope[n] = 0.f;
252			}
253			}
254		✗	}
255
256		✗	static void get_factors(AVFilterContext *ctx,
257			int len,
258			float *factors,
259			const float *envelope)
260			{
261		✗	for (int n = 0; n < len; n++)
262		✗	factors[n] = sqrtf(ff_exp10f(envelope[n] / 10.f));
263		✗	}
264
265		✗	static void apply_factors(AVFilterContext *ctx,
266			int len,
267			float *spectrum,
268			const float *factors)
269			{
270		✗	for (int n = 0; n < len; n++) {
271		✗	spectrum[2*n+0] *= factors[n];
272		✗	spectrum[2*n+1] *= factors[n];
273			}
274		✗	}
275
276		✗	static void feed(AVFilterContext *ctx, int ch,
277			const float *in_samples, float *out_samples,
278			float *in_frame, float *out_dist_frame,
279			float *windowed_frame, float *drc_frame,
280			float *spectrum_buf, float *energy,
281			float *target_gain, float *envelope,
282			float *factors)
283			{
284		✗	AudioDRCContext *s = ctx->priv;
285			double var_values[VAR_VARS_NB];
286		✗	const int fft_size = s->fft_size;
287		✗	const int nb_coeffs = s->fft_size / 2 + 1;
288		✗	const int overlap = s->overlap;
289		✗	enum AVChannel channel = av_channel_layout_channel_from_index(&ctx->inputs[0]->ch_layout, ch);
290		✗	const int bypass = av_channel_layout_index_from_channel(&s->ch_layout, channel) < 0;
291
292		✗	memcpy(var_values, s->var_values, sizeof(var_values));
293
294		✗	var_values[VAR_CH] = ch;
295
296			// shift in/out buffers
297		✗	memmove(in_frame, in_frame + overlap, (fft_size - overlap) * sizeof(*in_frame));
298		✗	memmove(out_dist_frame, out_dist_frame + overlap, (fft_size - overlap) * sizeof(*out_dist_frame));
299
300		✗	memcpy(in_frame + fft_size - overlap, in_samples, sizeof(*in_frame) * overlap);
301		✗	memset(out_dist_frame + fft_size - overlap, 0, sizeof(*out_dist_frame) * overlap);
302
303		✗	apply_window(s, in_frame, windowed_frame, 0);
304		✗	s->tx_fn(s->tx_ctx[ch], spectrum_buf, windowed_frame, sizeof(float));
305
306		✗	get_energy(ctx, nb_coeffs, energy, spectrum_buf);
307		✗	get_target_gain(ctx, nb_coeffs, target_gain, energy, var_values, s->fx, bypass);
308		✗	get_envelope(ctx, nb_coeffs, envelope, energy, target_gain);
309		✗	get_factors(ctx, nb_coeffs, factors, envelope);
310		✗	apply_factors(ctx, nb_coeffs, spectrum_buf, factors);
311
312		✗	s->itx_fn(s->itx_ctx[ch], drc_frame, spectrum_buf, sizeof(AVComplexFloat));
313
314		✗	apply_window(s, drc_frame, out_dist_frame, 1);
315
316			// 4 times overlap with squared hanning window results in 1.5 time increase in amplitude
317		✗	if (!ctx->is_disabled) {
318		✗	for (int i = 0; i < overlap; i++)
319		✗	out_samples[i] = out_dist_frame[i] / 1.5f;
320			} else {
321		✗	memcpy(out_samples, in_frame, sizeof(*out_samples) * overlap);
322			}
323		✗	}
324
325		✗	static int drc_channel(AVFilterContext *ctx, AVFrame *in, AVFrame *out, int ch)
326			{
327		✗	AudioDRCContext *s = ctx->priv;
328		✗	const float *src = (const float *)in->extended_data[ch];
329		✗	float *in_buffer = (float *)s->in_buffer->extended_data[ch];
330		✗	float *dst = (float *)out->extended_data[ch];
331
332		✗	memcpy(in_buffer, src, sizeof(*in_buffer) * s->overlap);
333
334		✗	feed(ctx, ch, in_buffer, dst,
335		✗	(float *)(s->in_frame->extended_data[ch]),
336		✗	(float *)(s->out_dist_frame->extended_data[ch]),
337		✗	(float *)(s->windowed_frame->extended_data[ch]),
338		✗	(float *)(s->drc_frame->extended_data[ch]),
339		✗	(float *)(s->spectrum_buf->extended_data[ch]),
340		✗	(float *)(s->energy->extended_data[ch]),
341		✗	(float *)(s->target_gain->extended_data[ch]),
342		✗	(float *)(s->envelope->extended_data[ch]),
343		✗	(float *)(s->factors->extended_data[ch]));
344
345		✗	return 0;
346			}
347
348		✗	static int drc_channels(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
349			{
350		✗	AudioDRCContext *s = ctx->priv;
351		✗	AVFrame *in = s->in;
352		✗	AVFrame *out = arg;
353		✗	const int start = (out->ch_layout.nb_channels * jobnr) / nb_jobs;
354		✗	const int end = (out->ch_layout.nb_channels * (jobnr+1)) / nb_jobs;
355
356		✗	for (int ch = start; ch < end; ch++)
357		✗	drc_channel(ctx, in, out, ch);
358
359		✗	return 0;
360			}
361
362		✗	static int filter_frame(AVFilterLink *inlink, AVFrame *in)
363			{
364		✗	AVFilterContext *ctx = inlink->dst;
365		✗	AVFilterLink *outlink = ctx->outputs[0];
366		✗	AudioDRCContext *s = ctx->priv;
367			AVFrame *out;
368			int ret;
369
370		✗	out = ff_get_audio_buffer(outlink, s->overlap);
371		✗	if (!out) {
372		✗	ret = AVERROR(ENOMEM);
373		✗	goto fail;
374			}
375
376		✗	s->var_values[VAR_SN] = outlink->sample_count_in;
377		✗	s->var_values[VAR_T] = s->var_values[VAR_SN] * (double)1/outlink->sample_rate;
378
379		✗	s->in = in;
380		✗	av_frame_copy_props(out, in);
381		✗	ff_filter_execute(ctx, drc_channels, out, NULL,
382		✗	FFMIN(outlink->ch_layout.nb_channels, ff_filter_get_nb_threads(ctx)));
383
384		✗	out->pts = in->pts;
385		✗	out->nb_samples = in->nb_samples;
386		✗	ret = ff_filter_frame(outlink, out);
387		✗	fail:
388		✗	av_frame_free(&in);
389		✗	s->in = NULL;
390		✗	return ret < 0 ? ret : 0;
391			}
392
393		✗	static int activate(AVFilterContext *ctx)
394			{
395		✗	AVFilterLink *inlink = ctx->inputs[0];
396		✗	AVFilterLink *outlink = ctx->outputs[0];
397		✗	AudioDRCContext *s = ctx->priv;
398		✗	AVFrame *in = NULL;
399		✗	int ret = 0, status;
400			int64_t pts;
401
402		✗	ret = av_channel_layout_copy(&s->ch_layout, &inlink->ch_layout);
403		✗	if (ret < 0)
404		✗	return ret;
405		✗	if (strcmp(s->channels_to_filter, "all"))
406		✗	av_channel_layout_from_string(&s->ch_layout, s->channels_to_filter);
407
408		✗	FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
409
410		✗	ret = ff_inlink_consume_samples(inlink, s->overlap, s->overlap, &in);
411		✗	if (ret < 0)
412		✗	return ret;
413
414		✗	if (ret > 0) {
415		✗	s->attack = expf(-1.f / (s->attack_ms * inlink->sample_rate / 1000.f));
416		✗	s->release = expf(-1.f / (s->release_ms * inlink->sample_rate / 1000.f));
417
418		✗	return filter_frame(inlink, in);
419		✗	} else if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
420		✗	ff_outlink_set_status(outlink, status, pts);
421		✗	return 0;
422			} else {
423		✗	if (ff_inlink_queued_samples(inlink) >= s->overlap) {
424		✗	ff_filter_set_ready(ctx, 10);
425		✗	} else if (ff_outlink_frame_wanted(outlink)) {
426		✗	ff_inlink_request_frame(inlink);
427			}
428		✗	return 0;
429			}
430			}
431
432		✗	static av_cold void uninit(AVFilterContext *ctx)
433			{
434		✗	AudioDRCContext *s = ctx->priv;
435
436		✗	av_channel_layout_uninit(&s->ch_layout);
437
438		✗	av_expr_free(s->expr);
439		✗	s->expr = NULL;
440
441		✗	av_freep(&s->window);
442
443		✗	av_frame_free(&s->drc_frame);
444		✗	av_frame_free(&s->energy);
445		✗	av_frame_free(&s->envelope);
446		✗	av_frame_free(&s->factors);
447		✗	av_frame_free(&s->in_buffer);
448		✗	av_frame_free(&s->in_frame);
449		✗	av_frame_free(&s->out_dist_frame);
450		✗	av_frame_free(&s->spectrum_buf);
451		✗	av_frame_free(&s->target_gain);
452		✗	av_frame_free(&s->windowed_frame);
453
454		✗	for (int ch = 0; ch < s->channels; ch++) {
455		✗	if (s->tx_ctx)
456		✗	av_tx_uninit(&s->tx_ctx[ch]);
457		✗	if (s->itx_ctx)
458		✗	av_tx_uninit(&s->itx_ctx[ch]);
459			}
460
461		✗	av_freep(&s->tx_ctx);
462		✗	av_freep(&s->itx_ctx);
463		✗	}
464
465		✗	static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
466			char *res, int res_len, int flags)
467			{
468		✗	AudioDRCContext *s = ctx->priv;
469		✗	char *old_expr_str = av_strdup(s->expr_str);
470			int ret;
471
472		✗	ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags);
473		✗	if (ret >= 0 && strcmp(old_expr_str, s->expr_str)) {
474		✗	ret = av_expr_parse(&s->expr, s->expr_str, var_names, NULL, NULL,
475			NULL, NULL, 0, ctx);
476			}
477		✗	av_free(old_expr_str);
478		✗	return ret;
479			}
480
481			static const AVFilterPad inputs[] = {
482			{
483			.name = "default",
484			.type = AVMEDIA_TYPE_AUDIO,
485			.config_props = config_input,
486			},
487			};
488
489			const AVFilter ff_af_adrc = {
490			.name = "adrc",
491			.description = NULL_IF_CONFIG_SMALL("Audio Spectral Dynamic Range Controller."),
492			.priv_size = sizeof(AudioDRCContext),
493			.priv_class = &adrc_class,
494			.uninit = uninit,
495			FILTER_INPUTS(inputs),
496			FILTER_OUTPUTS(ff_audio_default_filterpad),
497			FILTER_SINGLE_SAMPLEFMT(AV_SAMPLE_FMT_FLTP),
498			.flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL |
499			AVFILTER_FLAG_SLICE_THREADS,
500			.activate = activate,
501			.process_command = process_command,
502			};
503