FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2016 Paul B Mahol
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or
7			* modify it under the terms of the GNU Lesser General Public
8			* License as published by the Free Software Foundation; either
9			* version 2.1 of the License, or (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14			* Lesser General Public License for more details.
15			*
16			* You should have received a copy of the GNU Lesser General Public
17			* License along with FFmpeg; if not, write to the Free Software
18			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19			*/
20
21			/**
22			* @file
23			* SpectrumSynth filter
24			* @todo support float pixel format
25			*/
26
27			#include "libavutil/mem.h"
28			#include "libavutil/tx.h"
29			#include "libavutil/avassert.h"
30			#include "libavutil/cpu.h"
31			#include "libavutil/ffmath.h"
32			#include "libavutil/opt.h"
33			#include "avfilter.h"
34			#include "formats.h"
35			#include "audio.h"
36			#include "filters.h"
37			#include "internal.h"
38			#include "window_func.h"
39
40			enum MagnitudeScale { LINEAR, LOG, NB_SCALES };
41			enum SlideMode { REPLACE, SCROLL, FULLFRAME, RSCROLL, NB_SLIDES };
42			enum Orientation { VERTICAL, HORIZONTAL, NB_ORIENTATIONS };
43
44			typedef struct SpectrumSynthContext {
45			const AVClass *class;
46			int sample_rate;
47			int channels;
48			int scale;
49			int sliding;
50			int win_func;
51			float overlap;
52			int orientation;
53
54			AVFrame *magnitude, *phase;
55			AVTXContext *fft; ///< Fast Fourier Transform context
56			av_tx_fn tx_fn;
57			AVComplexFloat **fft_in; ///< bins holder for each (displayed) channels
58			AVComplexFloat **fft_out; ///< bins holder for each (displayed) channels
59			int win_size;
60			int size;
61			int nb_freq;
62			int hop_size;
63			int start, end;
64			int xpos;
65			int xend;
66			int64_t pts;
67			float factor;
68			AVFrame *buffer;
69			float *window_func_lut; ///< Window function LUT
70			} SpectrumSynthContext;
71
72			#define OFFSET(x) offsetof(SpectrumSynthContext, x)
73			#define A AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_AUDIO_PARAM
74			#define V AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
75
76			static const AVOption spectrumsynth_options[] = {
77			{ "sample_rate", "set sample rate", OFFSET(sample_rate), AV_OPT_TYPE_INT, {.i64 = 44100}, 15, INT_MAX, A },
78			{ "channels", "set channels", OFFSET(channels), AV_OPT_TYPE_INT, {.i64 = 1}, 1, 8, A },
79			{ "scale", "set input amplitude scale", OFFSET(scale), AV_OPT_TYPE_INT, {.i64 = LOG}, 0, NB_SCALES-1, V, .unit = "scale" },
80			{ "lin", "linear", 0, AV_OPT_TYPE_CONST, {.i64=LINEAR}, 0, 0, V, .unit = "scale" },
81			{ "log", "logarithmic", 0, AV_OPT_TYPE_CONST, {.i64=LOG}, 0, 0, V, .unit = "scale" },
82			{ "slide", "set input sliding mode", OFFSET(sliding), AV_OPT_TYPE_INT, {.i64 = FULLFRAME}, 0, NB_SLIDES-1, V, .unit = "slide" },
83			{ "replace", "consume old columns with new", 0, AV_OPT_TYPE_CONST, {.i64=REPLACE}, 0, 0, V, .unit = "slide" },
84			{ "scroll", "consume only most right column", 0, AV_OPT_TYPE_CONST, {.i64=SCROLL}, 0, 0, V, .unit = "slide" },
85			{ "fullframe", "consume full frames", 0, AV_OPT_TYPE_CONST, {.i64=FULLFRAME}, 0, 0, V, .unit = "slide" },
86			{ "rscroll", "consume only most left column", 0, AV_OPT_TYPE_CONST, {.i64=RSCROLL}, 0, 0, V, .unit = "slide" },
87			WIN_FUNC_OPTION("win_func", OFFSET(win_func), A, 0),
88			{ "overlap", "set window overlap", OFFSET(overlap), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0, 1, A },
89			{ "orientation", "set orientation", OFFSET(orientation), AV_OPT_TYPE_INT, {.i64=VERTICAL}, 0, NB_ORIENTATIONS-1, V, .unit = "orientation" },
90			{ "vertical", NULL, 0, AV_OPT_TYPE_CONST, {.i64=VERTICAL}, 0, 0, V, .unit = "orientation" },
91			{ "horizontal", NULL, 0, AV_OPT_TYPE_CONST, {.i64=HORIZONTAL}, 0, 0, V, .unit = "orientation" },
92			{ NULL }
93			};
94
95			AVFILTER_DEFINE_CLASS(spectrumsynth);
96
97		✗	static int query_formats(AVFilterContext *ctx)
98			{
99		✗	SpectrumSynthContext *s = ctx->priv;
100		✗	AVFilterFormats *formats = NULL;
101		✗	AVFilterChannelLayouts *layout = NULL;
102		✗	AVFilterLink *magnitude = ctx->inputs[0];
103		✗	AVFilterLink *phase = ctx->inputs[1];
104		✗	AVFilterLink *outlink = ctx->outputs[0];
105			static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE };
106			static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY16,
107			AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ444P,
108			AV_PIX_FMT_YUV444P16, AV_PIX_FMT_NONE };
109		✗	int ret, sample_rates[] = { 48000, -1 };
110
111		✗	formats = ff_make_format_list(sample_fmts);
112		✗	if ((ret = ff_formats_ref (formats, &outlink->incfg.formats )) < 0 ||
113		✗	(ret = ff_add_channel_layout (&layout, &FF_COUNT2LAYOUT(s->channels))) < 0 ||
114		✗	(ret = ff_channel_layouts_ref (layout , &outlink->incfg.channel_layouts)) < 0)
115		✗	return ret;
116
117		✗	sample_rates[0] = s->sample_rate;
118		✗	formats = ff_make_format_list(sample_rates);
119		✗	if (!formats)
120		✗	return AVERROR(ENOMEM);
121		✗	if ((ret = ff_formats_ref(formats, &outlink->incfg.samplerates)) < 0)
122		✗	return ret;
123
124		✗	formats = ff_make_format_list(pix_fmts);
125		✗	if (!formats)
126		✗	return AVERROR(ENOMEM);
127		✗	if ((ret = ff_formats_ref(formats, &magnitude->outcfg.formats)) < 0)
128		✗	return ret;
129
130		✗	formats = ff_make_format_list(pix_fmts);
131		✗	if (!formats)
132		✗	return AVERROR(ENOMEM);
133		✗	if ((ret = ff_formats_ref(formats, &phase->outcfg.formats)) < 0)
134		✗	return ret;
135
136		✗	return 0;
137			}
138
139		✗	static int config_output(AVFilterLink *outlink)
140			{
141		✗	AVFilterContext *ctx = outlink->src;
142		✗	SpectrumSynthContext *s = ctx->priv;
143		✗	int width = ctx->inputs[0]->w;
144		✗	int height = ctx->inputs[0]->h;
145		✗	AVRational time_base = ctx->inputs[0]->time_base;
146		✗	AVRational frame_rate = ctx->inputs[0]->frame_rate;
147			float factor, overlap, scale;
148			int i, ch, ret;
149
150		✗	outlink->sample_rate = s->sample_rate;
151		✗	outlink->time_base = (AVRational){1, s->sample_rate};
152
153		✗	if (width != ctx->inputs[1]->w ||
154		✗	height != ctx->inputs[1]->h) {
155		✗	av_log(ctx, AV_LOG_ERROR,
156			"Magnitude and Phase sizes differ (%dx%d vs %dx%d).\n",
157			width, height,
158		✗	ctx->inputs[1]->w, ctx->inputs[1]->h);
159		✗	return AVERROR_INVALIDDATA;
160		✗	} else if (av_cmp_q(time_base, ctx->inputs[1]->time_base) != 0) {
161		✗	av_log(ctx, AV_LOG_ERROR,
162			"Magnitude and Phase time bases differ (%d/%d vs %d/%d).\n",
163			time_base.num, time_base.den,
164		✗	ctx->inputs[1]->time_base.num,
165		✗	ctx->inputs[1]->time_base.den);
166		✗	return AVERROR_INVALIDDATA;
167		✗	} else if (av_cmp_q(frame_rate, ctx->inputs[1]->frame_rate) != 0) {
168		✗	av_log(ctx, AV_LOG_ERROR,
169			"Magnitude and Phase framerates differ (%d/%d vs %d/%d).\n",
170			frame_rate.num, frame_rate.den,
171		✗	ctx->inputs[1]->frame_rate.num,
172		✗	ctx->inputs[1]->frame_rate.den);
173		✗	return AVERROR_INVALIDDATA;
174			}
175
176		✗	s->size = s->orientation == VERTICAL ? height / s->channels : width / s->channels;
177		✗	s->xend = s->orientation == VERTICAL ? width : height;
178
179		✗	s->win_size = s->size * 2;
180		✗	s->nb_freq = s->size;
181
182		✗	ret = av_tx_init(&s->fft, &s->tx_fn, AV_TX_FLOAT_FFT, 1, s->win_size, &scale, 0);
183		✗	if (ret < 0) {
184		✗	av_log(ctx, AV_LOG_ERROR, "Unable to create FFT context. "
185			"The window size might be too high.\n");
186		✗	return ret;
187			}
188
189		✗	s->fft_in = av_calloc(s->channels, sizeof(*s->fft_in));
190		✗	if (!s->fft_in)
191		✗	return AVERROR(ENOMEM);
192		✗	s->fft_out = av_calloc(s->channels, sizeof(*s->fft_out));
193		✗	if (!s->fft_out)
194		✗	return AVERROR(ENOMEM);
195
196		✗	for (ch = 0; ch < s->channels; ch++) {
197		✗	s->fft_in[ch] = av_calloc(FFALIGN(s->win_size, av_cpu_max_align()), sizeof(**s->fft_in));
198		✗	if (!s->fft_in[ch])
199		✗	return AVERROR(ENOMEM);
200
201		✗	s->fft_out[ch] = av_calloc(FFALIGN(s->win_size, av_cpu_max_align()), sizeof(**s->fft_out));
202		✗	if (!s->fft_out[ch])
203		✗	return AVERROR(ENOMEM);
204			}
205
206		✗	s->buffer = ff_get_audio_buffer(outlink, s->win_size * 2);
207		✗	if (!s->buffer)
208		✗	return AVERROR(ENOMEM);
209
210			/* pre-calc windowing function */
211		✗	s->window_func_lut = av_realloc_f(s->window_func_lut, s->win_size,
212			sizeof(*s->window_func_lut));
213		✗	if (!s->window_func_lut)
214		✗	return AVERROR(ENOMEM);
215		✗	generate_window_func(s->window_func_lut, s->win_size, s->win_func, &overlap);
216		✗	if (s->overlap == 1)
217		✗	s->overlap = overlap;
218		✗	s->hop_size = (1 - s->overlap) * s->win_size;
219		✗	for (factor = 0, i = 0; i < s->win_size; i++) {
220		✗	factor += s->window_func_lut[i] * s->window_func_lut[i];
221			}
222		✗	s->factor = (factor / s->win_size) / FFMAX(1 / (1 - s->overlap) - 1, 1);
223
224		✗	return 0;
225			}
226
227		✗	static void read16_fft_bin(SpectrumSynthContext *s,
228			int x, int y, int f, int ch)
229			{
230		✗	const int m_linesize = s->magnitude->linesize[0];
231		✗	const int p_linesize = s->phase->linesize[0];
232		✗	const uint16_t *m = (uint16_t *)(s->magnitude->data[0] + y * m_linesize);
233		✗	const uint16_t *p = (uint16_t *)(s->phase->data[0] + y * p_linesize);
234			float magnitude, phase;
235
236		✗	switch (s->scale) {
237		✗	case LINEAR:
238		✗	magnitude = m[x] / (double)UINT16_MAX;
239		✗	break;
240		✗	case LOG:
241		✗	magnitude = ff_exp10(((m[x] / (double)UINT16_MAX) - 1.) * 6.);
242		✗	break;
243		✗	default:
244		✗	av_assert0(0);
245			}
246		✗	phase = ((p[x] / (double)UINT16_MAX) * 2. - 1.) * M_PI;
247
248		✗	s->fft_in[ch][f].re = magnitude * cos(phase);
249		✗	s->fft_in[ch][f].im = magnitude * sin(phase);
250		✗	}
251
252		✗	static void read8_fft_bin(SpectrumSynthContext *s,
253			int x, int y, int f, int ch)
254			{
255		✗	const int m_linesize = s->magnitude->linesize[0];
256		✗	const int p_linesize = s->phase->linesize[0];
257		✗	const uint8_t *m = (uint8_t *)(s->magnitude->data[0] + y * m_linesize);
258		✗	const uint8_t *p = (uint8_t *)(s->phase->data[0] + y * p_linesize);
259			float magnitude, phase;
260
261		✗	switch (s->scale) {
262		✗	case LINEAR:
263		✗	magnitude = m[x] / (double)UINT8_MAX;
264		✗	break;
265		✗	case LOG:
266		✗	magnitude = ff_exp10(((m[x] / (double)UINT8_MAX) - 1.) * 6.);
267		✗	break;
268		✗	default:
269		✗	av_assert0(0);
270			}
271		✗	phase = ((p[x] / (double)UINT8_MAX) * 2. - 1.) * M_PI;
272
273		✗	s->fft_in[ch][f].re = magnitude * cos(phase);
274		✗	s->fft_in[ch][f].im = magnitude * sin(phase);
275		✗	}
276
277		✗	static void read_fft_data(AVFilterContext *ctx, int x, int h, int ch)
278			{
279		✗	SpectrumSynthContext *s = ctx->priv;
280		✗	AVFilterLink *inlink = ctx->inputs[0];
281		✗	int start = h * (s->channels - ch) - 1;
282		✗	int end = h * (s->channels - ch - 1);
283			int y, f;
284
285		✗	switch (s->orientation) {
286		✗	case VERTICAL:
287		✗	switch (inlink->format) {
288		✗	case AV_PIX_FMT_YUV444P16:
289			case AV_PIX_FMT_GRAY16:
290		✗	for (y = start, f = 0; y >= end; y--, f++) {
291		✗	read16_fft_bin(s, x, y, f, ch);
292			}
293		✗	break;
294		✗	case AV_PIX_FMT_YUVJ444P:
295			case AV_PIX_FMT_YUV444P:
296			case AV_PIX_FMT_GRAY8:
297		✗	for (y = start, f = 0; y >= end; y--, f++) {
298		✗	read8_fft_bin(s, x, y, f, ch);
299			}
300		✗	break;
301			}
302		✗	break;
303		✗	case HORIZONTAL:
304		✗	switch (inlink->format) {
305		✗	case AV_PIX_FMT_YUV444P16:
306			case AV_PIX_FMT_GRAY16:
307		✗	for (y = end, f = 0; y <= start; y++, f++) {
308		✗	read16_fft_bin(s, y, x, f, ch);
309			}
310		✗	break;
311		✗	case AV_PIX_FMT_YUVJ444P:
312			case AV_PIX_FMT_YUV444P:
313			case AV_PIX_FMT_GRAY8:
314		✗	for (y = end, f = 0; y <= start; y++, f++) {
315		✗	read8_fft_bin(s, y, x, f, ch);
316			}
317		✗	break;
318			}
319		✗	break;
320			}
321		✗	}
322
323		✗	static void synth_window(AVFilterContext *ctx, int x)
324			{
325		✗	SpectrumSynthContext *s = ctx->priv;
326		✗	const int h = s->size;
327		✗	int nb = s->win_size;
328			int y, f, ch;
329
330		✗	for (ch = 0; ch < s->channels; ch++) {
331		✗	read_fft_data(ctx, x, h, ch);
332
333		✗	for (y = h; y <= s->nb_freq; y++) {
334		✗	s->fft_in[ch][y].re = 0;
335		✗	s->fft_in[ch][y].im = 0;
336			}
337
338		✗	for (y = s->nb_freq + 1, f = s->nb_freq - 1; y < nb; y++, f--) {
339		✗	s->fft_in[ch][y].re = s->fft_in[ch][f].re;
340		✗	s->fft_in[ch][y].im = -s->fft_in[ch][f].im;
341			}
342
343		✗	s->tx_fn(s->fft, s->fft_out[ch], s->fft_in[ch], sizeof(AVComplexFloat));
344			}
345		✗	}
346
347		✗	static int try_push_frame(AVFilterContext *ctx, int x)
348			{
349		✗	SpectrumSynthContext *s = ctx->priv;
350		✗	AVFilterLink *outlink = ctx->outputs[0];
351		✗	const float factor = s->factor;
352			int ch, n, i, ret;
353			int start, end;
354			AVFrame *out;
355
356		✗	synth_window(ctx, x);
357
358		✗	for (ch = 0; ch < s->channels; ch++) {
359		✗	float *buf = (float *)s->buffer->extended_data[ch];
360			int j, k;
361
362		✗	start = s->start;
363		✗	end = s->end;
364		✗	k = end;
365		✗	for (i = 0, j = start; j < k && i < s->win_size; i++, j++) {
366		✗	buf[j] += s->fft_out[ch][i].re;
367			}
368
369		✗	for (; i < s->win_size; i++, j++) {
370		✗	buf[j] = s->fft_out[ch][i].re;
371			}
372
373		✗	start += s->hop_size;
374		✗	end = j;
375
376		✗	if (start >= s->win_size) {
377		✗	start -= s->win_size;
378		✗	end -= s->win_size;
379
380		✗	if (ch == s->channels - 1) {
381			float *dst;
382			int c;
383
384		✗	out = ff_get_audio_buffer(outlink, s->win_size);
385		✗	if (!out) {
386		✗	av_frame_free(&s->magnitude);
387		✗	av_frame_free(&s->phase);
388		✗	return AVERROR(ENOMEM);
389			}
390
391		✗	out->pts = s->pts;
392		✗	s->pts += s->win_size;
393		✗	for (c = 0; c < s->channels; c++) {
394		✗	dst = (float *)out->extended_data[c];
395		✗	buf = (float *)s->buffer->extended_data[c];
396
397		✗	for (n = 0; n < s->win_size; n++) {
398		✗	dst[n] = buf[n] * factor;
399			}
400		✗	memmove(buf, buf + s->win_size, s->win_size * 4);
401			}
402
403		✗	ret = ff_filter_frame(outlink, out);
404		✗	if (ret < 0)
405		✗	return ret;
406			}
407			}
408			}
409
410		✗	s->start = start;
411		✗	s->end = end;
412
413		✗	return 0;
414			}
415
416		✗	static int try_push_frames(AVFilterContext *ctx)
417			{
418		✗	SpectrumSynthContext *s = ctx->priv;
419			int ret, x;
420
421		✗	if (!(s->magnitude && s->phase))
422		✗	return 0;
423
424		✗	switch (s->sliding) {
425		✗	case REPLACE:
426		✗	ret = try_push_frame(ctx, s->xpos);
427		✗	s->xpos++;
428		✗	if (s->xpos >= s->xend)
429		✗	s->xpos = 0;
430		✗	break;
431		✗	case SCROLL:
432		✗	s->xpos = s->xend - 1;
433		✗	ret = try_push_frame(ctx, s->xpos);
434		✗	break;
435		✗	case RSCROLL:
436		✗	s->xpos = 0;
437		✗	ret = try_push_frame(ctx, s->xpos);
438		✗	break;
439		✗	case FULLFRAME:
440		✗	for (x = 0; x < s->xend; x++) {
441		✗	ret = try_push_frame(ctx, x);
442		✗	if (ret < 0)
443		✗	break;
444			}
445		✗	break;
446		✗	default:
447		✗	av_assert0(0);
448			}
449
450		✗	av_frame_free(&s->magnitude);
451		✗	av_frame_free(&s->phase);
452		✗	return ret;
453			}
454
455		✗	static int activate(AVFilterContext *ctx)
456			{
457		✗	SpectrumSynthContext *s = ctx->priv;
458		✗	AVFrame **staging[2] = { &s->magnitude, &s->phase };
459			int64_t pts;
460			int i, ret;
461
462		✗	FF_FILTER_FORWARD_STATUS_BACK_ALL(ctx->outputs[0], ctx);
463
464		✗	for (i = 0; i < 2; i++) {
465		✗	if (*staging[i])
466		✗	continue;
467		✗	ret = ff_inlink_consume_frame(ctx->inputs[i], staging[i]);
468		✗	if (ret < 0)
469		✗	return ret;
470		✗	if (ret) {
471		✗	ff_filter_set_ready(ctx, 10);
472		✗	return try_push_frames(ctx);
473			}
474			}
475
476		✗	for (i = 0; i < 2; i++) {
477		✗	if (ff_inlink_acknowledge_status(ctx->inputs[i], &ret, &pts)) {
478		✗	ff_outlink_set_status(ctx->outputs[0], ret, pts);
479		✗	ff_inlink_set_status(ctx->inputs[1 - i], ret);
480		✗	return 0;
481			}
482			}
483
484		✗	if (ff_outlink_frame_wanted(ctx->outputs[0])) {
485		✗	for (i = 0; i < 2; i++) {
486		✗	if (!*staging[i])
487		✗	ff_inlink_request_frame(ctx->inputs[i]);
488			}
489			}
490
491		✗	return FFERROR_NOT_READY;
492			}
493
494		✗	static av_cold void uninit(AVFilterContext *ctx)
495			{
496		✗	SpectrumSynthContext *s = ctx->priv;
497			int i;
498
499		✗	av_frame_free(&s->magnitude);
500		✗	av_frame_free(&s->phase);
501		✗	av_frame_free(&s->buffer);
502
503		✗	av_tx_uninit(&s->fft);
504
505		✗	if (s->fft_in) {
506		✗	for (i = 0; i < s->channels; i++)
507		✗	av_freep(&s->fft_in[i]);
508			}
509		✗	if (s->fft_out) {
510		✗	for (i = 0; i < s->channels; i++)
511		✗	av_freep(&s->fft_out[i]);
512			}
513		✗	av_freep(&s->fft_in);
514		✗	av_freep(&s->fft_out);
515		✗	av_freep(&s->window_func_lut);
516		✗	}
517
518			static const AVFilterPad spectrumsynth_inputs[] = {
519			{
520			.name = "magnitude",
521			.type = AVMEDIA_TYPE_VIDEO,
522			},
523			{
524			.name = "phase",
525			.type = AVMEDIA_TYPE_VIDEO,
526			},
527			};
528
529			static const AVFilterPad spectrumsynth_outputs[] = {
530			{
531			.name = "default",
532			.type = AVMEDIA_TYPE_AUDIO,
533			.config_props = config_output,
534			},
535			};
536
537			const AVFilter ff_vaf_spectrumsynth = {
538			.name = "spectrumsynth",
539			.description = NULL_IF_CONFIG_SMALL("Convert input spectrum videos to audio output."),
540			.uninit = uninit,
541			.activate = activate,
542			.priv_size = sizeof(SpectrumSynthContext),
543			FILTER_INPUTS(spectrumsynth_inputs),
544			FILTER_OUTPUTS(spectrumsynth_outputs),
545			FILTER_QUERY_FUNC(query_formats),
546			.priv_class = &spectrumsynth_class,
547			};
548