FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2016 Paul B Mahol
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or
7			* modify it under the terms of the GNU Lesser General Public
8			* License as published by the Free Software Foundation; either
9			* version 2.1 of the License, or (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14			* Lesser General Public License for more details.
15			*
16			* You should have received a copy of the GNU Lesser General Public
17			* License along with FFmpeg; if not, write to the Free Software
18			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19			*/
20
21			/**
22			* @file
23			* SpectrumSynth filter
24			* @todo support float pixel format
25			*/
26
27			#include "libavutil/mem.h"
28			#include "libavutil/tx.h"
29			#include "libavutil/avassert.h"
30			#include "libavutil/cpu.h"
31			#include "libavutil/ffmath.h"
32			#include "libavutil/opt.h"
33			#include "avfilter.h"
34			#include "formats.h"
35			#include "audio.h"
36			#include "filters.h"
37			#include "window_func.h"
38
39			enum MagnitudeScale { LINEAR, LOG, NB_SCALES };
40			enum SlideMode { REPLACE, SCROLL, FULLFRAME, RSCROLL, NB_SLIDES };
41			enum Orientation { VERTICAL, HORIZONTAL, NB_ORIENTATIONS };
42
43			typedef struct SpectrumSynthContext {
44			const AVClass *class;
45			int sample_rate;
46			int channels;
47			int scale;
48			int sliding;
49			int win_func;
50			float overlap;
51			int orientation;
52
53			AVFrame *magnitude, *phase;
54			AVTXContext *fft; ///< Fast Fourier Transform context
55			av_tx_fn tx_fn;
56			AVComplexFloat **fft_in; ///< bins holder for each (displayed) channels
57			AVComplexFloat **fft_out; ///< bins holder for each (displayed) channels
58			int win_size;
59			int size;
60			int nb_freq;
61			int hop_size;
62			int start, end;
63			int xpos;
64			int xend;
65			int64_t pts;
66			float factor;
67			AVFrame *buffer;
68			float *window_func_lut; ///< Window function LUT
69			} SpectrumSynthContext;
70
71			#define OFFSET(x) offsetof(SpectrumSynthContext, x)
72			#define A AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_AUDIO_PARAM
73			#define V AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
74
75			static const AVOption spectrumsynth_options[] = {
76			{ "sample_rate", "set sample rate", OFFSET(sample_rate), AV_OPT_TYPE_INT, {.i64 = 44100}, 15, INT_MAX, A },
77			{ "channels", "set channels", OFFSET(channels), AV_OPT_TYPE_INT, {.i64 = 1}, 1, 8, A },
78			{ "scale", "set input amplitude scale", OFFSET(scale), AV_OPT_TYPE_INT, {.i64 = LOG}, 0, NB_SCALES-1, V, .unit = "scale" },
79			{ "lin", "linear", 0, AV_OPT_TYPE_CONST, {.i64=LINEAR}, 0, 0, V, .unit = "scale" },
80			{ "log", "logarithmic", 0, AV_OPT_TYPE_CONST, {.i64=LOG}, 0, 0, V, .unit = "scale" },
81			{ "slide", "set input sliding mode", OFFSET(sliding), AV_OPT_TYPE_INT, {.i64 = FULLFRAME}, 0, NB_SLIDES-1, V, .unit = "slide" },
82			{ "replace", "consume old columns with new", 0, AV_OPT_TYPE_CONST, {.i64=REPLACE}, 0, 0, V, .unit = "slide" },
83			{ "scroll", "consume only most right column", 0, AV_OPT_TYPE_CONST, {.i64=SCROLL}, 0, 0, V, .unit = "slide" },
84			{ "fullframe", "consume full frames", 0, AV_OPT_TYPE_CONST, {.i64=FULLFRAME}, 0, 0, V, .unit = "slide" },
85			{ "rscroll", "consume only most left column", 0, AV_OPT_TYPE_CONST, {.i64=RSCROLL}, 0, 0, V, .unit = "slide" },
86			WIN_FUNC_OPTION("win_func", OFFSET(win_func), A, 0),
87			{ "overlap", "set window overlap", OFFSET(overlap), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0, 1, A },
88			{ "orientation", "set orientation", OFFSET(orientation), AV_OPT_TYPE_INT, {.i64=VERTICAL}, 0, NB_ORIENTATIONS-1, V, .unit = "orientation" },
89			{ "vertical", NULL, 0, AV_OPT_TYPE_CONST, {.i64=VERTICAL}, 0, 0, V, .unit = "orientation" },
90			{ "horizontal", NULL, 0, AV_OPT_TYPE_CONST, {.i64=HORIZONTAL}, 0, 0, V, .unit = "orientation" },
91			{ NULL }
92			};
93
94			AVFILTER_DEFINE_CLASS(spectrumsynth);
95
96			enum {
97			MAGNITUDE = 0,
98			PHASE = 1,
99			};
100
101		✗	static int query_formats(const AVFilterContext *ctx,
102			AVFilterFormatsConfig **cfg_in,
103			AVFilterFormatsConfig **cfg_out)
104			{
105		✗	const SpectrumSynthContext *s = ctx->priv;
106		✗	AVFilterFormats *formats = NULL;
107		✗	AVFilterChannelLayouts *layout = NULL;
108			static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE };
109			static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY16,
110			AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ444P,
111			AV_PIX_FMT_YUV444P16, AV_PIX_FMT_NONE };
112		✗	int ret, sample_rates[] = { 48000, -1 };
113
114		✗	formats = ff_make_sample_format_list(sample_fmts);
115		✗	if ((ret = ff_formats_ref (formats, &cfg_out[0]->formats )) < 0 ||
116		✗	(ret = ff_add_channel_layout (&layout, &FF_COUNT2LAYOUT(s->channels))) < 0 ||
117		✗	(ret = ff_channel_layouts_ref (layout , &cfg_out[0]->channel_layouts)) < 0)
118		✗	return ret;
119
120		✗	sample_rates[0] = s->sample_rate;
121		✗	formats = ff_make_format_list(sample_rates);
122		✗	if (!formats)
123		✗	return AVERROR(ENOMEM);
124		✗	if ((ret = ff_formats_ref(formats, &cfg_out[0]->samplerates)) < 0)
125		✗	return ret;
126
127		✗	formats = ff_make_pixel_format_list(pix_fmts);
128		✗	if (!formats)
129		✗	return AVERROR(ENOMEM);
130		✗	if ((ret = ff_formats_ref(formats, &cfg_in[MAGNITUDE]->formats)) < 0)
131		✗	return ret;
132
133		✗	formats = ff_make_pixel_format_list(pix_fmts);
134		✗	if (!formats)
135		✗	return AVERROR(ENOMEM);
136		✗	if ((ret = ff_formats_ref(formats, &cfg_in[PHASE]->formats)) < 0)
137		✗	return ret;
138
139		✗	return 0;
140			}
141
142		✗	static int config_output(AVFilterLink *outlink)
143			{
144		✗	AVFilterContext *ctx = outlink->src;
145		✗	SpectrumSynthContext *s = ctx->priv;
146		✗	FilterLink *inl0 = ff_filter_link(ctx->inputs[0]);
147		✗	FilterLink *inl1 = ff_filter_link(ctx->inputs[1]);
148		✗	int width = ctx->inputs[0]->w;
149		✗	int height = ctx->inputs[0]->h;
150		✗	AVRational time_base = ctx->inputs[0]->time_base;
151		✗	AVRational frame_rate = inl0->frame_rate;
152			float factor, overlap;
153			int i, ch, ret;
154
155		✗	outlink->sample_rate = s->sample_rate;
156		✗	outlink->time_base = (AVRational){1, s->sample_rate};
157
158		✗	if (width != ctx->inputs[1]->w ||
159		✗	height != ctx->inputs[1]->h) {
160		✗	av_log(ctx, AV_LOG_ERROR,
161			"Magnitude and Phase sizes differ (%dx%d vs %dx%d).\n",
162			width, height,
163		✗	ctx->inputs[1]->w, ctx->inputs[1]->h);
164		✗	return AVERROR_INVALIDDATA;
165		✗	} else if (av_cmp_q(time_base, ctx->inputs[1]->time_base) != 0) {
166		✗	av_log(ctx, AV_LOG_ERROR,
167			"Magnitude and Phase time bases differ (%d/%d vs %d/%d).\n",
168			time_base.num, time_base.den,
169		✗	ctx->inputs[1]->time_base.num,
170		✗	ctx->inputs[1]->time_base.den);
171		✗	return AVERROR_INVALIDDATA;
172		✗	} else if (av_cmp_q(frame_rate, inl1->frame_rate) != 0) {
173		✗	av_log(ctx, AV_LOG_ERROR,
174			"Magnitude and Phase framerates differ (%d/%d vs %d/%d).\n",
175			frame_rate.num, frame_rate.den,
176			inl1->frame_rate.num,
177			inl1->frame_rate.den);
178		✗	return AVERROR_INVALIDDATA;
179			}
180
181		✗	s->size = s->orientation == VERTICAL ? height / s->channels : width / s->channels;
182		✗	s->xend = s->orientation == VERTICAL ? width : height;
183
184		✗	s->win_size = s->size * 2;
185		✗	s->nb_freq = s->size;
186
187		✗	ret = av_tx_init(&s->fft, &s->tx_fn, AV_TX_FLOAT_FFT, 1, s->win_size, NULL, 0);
188		✗	if (ret < 0) {
189		✗	av_log(ctx, AV_LOG_ERROR, "Unable to create FFT context. "
190			"The window size might be too high.\n");
191		✗	return ret;
192			}
193
194		✗	s->fft_in = av_calloc(s->channels, sizeof(*s->fft_in));
195		✗	if (!s->fft_in)
196		✗	return AVERROR(ENOMEM);
197		✗	s->fft_out = av_calloc(s->channels, sizeof(*s->fft_out));
198		✗	if (!s->fft_out)
199		✗	return AVERROR(ENOMEM);
200
201		✗	for (ch = 0; ch < s->channels; ch++) {
202		✗	s->fft_in[ch] = av_calloc(FFALIGN(s->win_size, av_cpu_max_align()), sizeof(**s->fft_in));
203		✗	if (!s->fft_in[ch])
204		✗	return AVERROR(ENOMEM);
205
206		✗	s->fft_out[ch] = av_calloc(FFALIGN(s->win_size, av_cpu_max_align()), sizeof(**s->fft_out));
207		✗	if (!s->fft_out[ch])
208		✗	return AVERROR(ENOMEM);
209			}
210
211		✗	s->buffer = ff_get_audio_buffer(outlink, s->win_size * 2);
212		✗	if (!s->buffer)
213		✗	return AVERROR(ENOMEM);
214
215			/* pre-calc windowing function */
216		✗	s->window_func_lut = av_realloc_f(s->window_func_lut, s->win_size,
217			sizeof(*s->window_func_lut));
218		✗	if (!s->window_func_lut)
219		✗	return AVERROR(ENOMEM);
220		✗	generate_window_func(s->window_func_lut, s->win_size, s->win_func, &overlap);
221		✗	if (s->overlap == 1)
222		✗	s->overlap = overlap;
223		✗	s->hop_size = (1 - s->overlap) * s->win_size;
224		✗	for (factor = 0, i = 0; i < s->win_size; i++) {
225		✗	factor += s->window_func_lut[i] * s->window_func_lut[i];
226			}
227		✗	s->factor = (factor / s->win_size) / FFMAX(1 / (1 - s->overlap) - 1, 1);
228
229		✗	return 0;
230			}
231
232		✗	static void read16_fft_bin(SpectrumSynthContext *s,
233			int x, int y, int f, int ch)
234			{
235		✗	const int m_linesize = s->magnitude->linesize[0];
236		✗	const int p_linesize = s->phase->linesize[0];
237		✗	const uint16_t *m = (uint16_t *)(s->magnitude->data[0] + y * m_linesize);
238		✗	const uint16_t *p = (uint16_t *)(s->phase->data[0] + y * p_linesize);
239			float magnitude, phase;
240
241		✗	switch (s->scale) {
242		✗	case LINEAR:
243		✗	magnitude = m[x] / (double)UINT16_MAX;
244		✗	break;
245		✗	case LOG:
246		✗	magnitude = ff_exp10(((m[x] / (double)UINT16_MAX) - 1.) * 6.);
247		✗	break;
248		✗	default:
249		✗	av_assert0(0);
250			}
251		✗	phase = ((p[x] / (double)UINT16_MAX) * 2. - 1.) * M_PI;
252
253		✗	s->fft_in[ch][f].re = magnitude * cos(phase);
254		✗	s->fft_in[ch][f].im = magnitude * sin(phase);
255		✗	}
256
257		✗	static void read8_fft_bin(SpectrumSynthContext *s,
258			int x, int y, int f, int ch)
259			{
260		✗	const int m_linesize = s->magnitude->linesize[0];
261		✗	const int p_linesize = s->phase->linesize[0];
262		✗	const uint8_t *m = (uint8_t *)(s->magnitude->data[0] + y * m_linesize);
263		✗	const uint8_t *p = (uint8_t *)(s->phase->data[0] + y * p_linesize);
264			float magnitude, phase;
265
266		✗	switch (s->scale) {
267		✗	case LINEAR:
268		✗	magnitude = m[x] / (double)UINT8_MAX;
269		✗	break;
270		✗	case LOG:
271		✗	magnitude = ff_exp10(((m[x] / (double)UINT8_MAX) - 1.) * 6.);
272		✗	break;
273		✗	default:
274		✗	av_assert0(0);
275			}
276		✗	phase = ((p[x] / (double)UINT8_MAX) * 2. - 1.) * M_PI;
277
278		✗	s->fft_in[ch][f].re = magnitude * cos(phase);
279		✗	s->fft_in[ch][f].im = magnitude * sin(phase);
280		✗	}
281
282		✗	static void read_fft_data(AVFilterContext *ctx, int x, int h, int ch)
283			{
284		✗	SpectrumSynthContext *s = ctx->priv;
285		✗	AVFilterLink *inlink = ctx->inputs[0];
286		✗	int start = h * (s->channels - ch) - 1;
287		✗	int end = h * (s->channels - ch - 1);
288			int y, f;
289
290		✗	switch (s->orientation) {
291		✗	case VERTICAL:
292		✗	switch (inlink->format) {
293		✗	case AV_PIX_FMT_YUV444P16:
294			case AV_PIX_FMT_GRAY16:
295		✗	for (y = start, f = 0; y >= end; y--, f++) {
296		✗	read16_fft_bin(s, x, y, f, ch);
297			}
298		✗	break;
299		✗	case AV_PIX_FMT_YUVJ444P:
300			case AV_PIX_FMT_YUV444P:
301			case AV_PIX_FMT_GRAY8:
302		✗	for (y = start, f = 0; y >= end; y--, f++) {
303		✗	read8_fft_bin(s, x, y, f, ch);
304			}
305		✗	break;
306			}
307		✗	break;
308		✗	case HORIZONTAL:
309		✗	switch (inlink->format) {
310		✗	case AV_PIX_FMT_YUV444P16:
311			case AV_PIX_FMT_GRAY16:
312		✗	for (y = end, f = 0; y <= start; y++, f++) {
313		✗	read16_fft_bin(s, y, x, f, ch);
314			}
315		✗	break;
316		✗	case AV_PIX_FMT_YUVJ444P:
317			case AV_PIX_FMT_YUV444P:
318			case AV_PIX_FMT_GRAY8:
319		✗	for (y = end, f = 0; y <= start; y++, f++) {
320		✗	read8_fft_bin(s, y, x, f, ch);
321			}
322		✗	break;
323			}
324		✗	break;
325			}
326		✗	}
327
328		✗	static void synth_window(AVFilterContext *ctx, int x)
329			{
330		✗	SpectrumSynthContext *s = ctx->priv;
331		✗	const int h = s->size;
332		✗	int nb = s->win_size;
333			int y, f, ch;
334
335		✗	for (ch = 0; ch < s->channels; ch++) {
336		✗	read_fft_data(ctx, x, h, ch);
337
338		✗	for (y = h; y <= s->nb_freq; y++) {
339		✗	s->fft_in[ch][y].re = 0;
340		✗	s->fft_in[ch][y].im = 0;
341			}
342
343		✗	for (y = s->nb_freq + 1, f = s->nb_freq - 1; y < nb; y++, f--) {
344		✗	s->fft_in[ch][y].re = s->fft_in[ch][f].re;
345		✗	s->fft_in[ch][y].im = -s->fft_in[ch][f].im;
346			}
347
348		✗	s->tx_fn(s->fft, s->fft_out[ch], s->fft_in[ch], sizeof(AVComplexFloat));
349			}
350		✗	}
351
352		✗	static int try_push_frame(AVFilterContext *ctx, int x)
353			{
354		✗	SpectrumSynthContext *s = ctx->priv;
355		✗	AVFilterLink *outlink = ctx->outputs[0];
356		✗	const float factor = s->factor;
357			int ch, n, i, ret;
358			int start, end;
359			AVFrame *out;
360
361		✗	synth_window(ctx, x);
362
363		✗	for (ch = 0; ch < s->channels; ch++) {
364		✗	float *buf = (float *)s->buffer->extended_data[ch];
365			int j, k;
366
367		✗	start = s->start;
368		✗	end = s->end;
369		✗	k = end;
370		✗	for (i = 0, j = start; j < k && i < s->win_size; i++, j++) {
371		✗	buf[j] += s->fft_out[ch][i].re;
372			}
373
374		✗	for (; i < s->win_size; i++, j++) {
375		✗	buf[j] = s->fft_out[ch][i].re;
376			}
377
378		✗	start += s->hop_size;
379		✗	end = j;
380
381		✗	if (start >= s->win_size) {
382		✗	start -= s->win_size;
383		✗	end -= s->win_size;
384
385		✗	if (ch == s->channels - 1) {
386			float *dst;
387			int c;
388
389		✗	out = ff_get_audio_buffer(outlink, s->win_size);
390		✗	if (!out) {
391		✗	av_frame_free(&s->magnitude);
392		✗	av_frame_free(&s->phase);
393		✗	return AVERROR(ENOMEM);
394			}
395
396		✗	out->pts = s->pts;
397		✗	s->pts += s->win_size;
398		✗	for (c = 0; c < s->channels; c++) {
399		✗	dst = (float *)out->extended_data[c];
400		✗	buf = (float *)s->buffer->extended_data[c];
401
402		✗	for (n = 0; n < s->win_size; n++) {
403		✗	dst[n] = buf[n] * factor;
404			}
405		✗	memmove(buf, buf + s->win_size, s->win_size * 4);
406			}
407
408		✗	ret = ff_filter_frame(outlink, out);
409		✗	if (ret < 0)
410		✗	return ret;
411			}
412			}
413			}
414
415		✗	s->start = start;
416		✗	s->end = end;
417
418		✗	return 0;
419			}
420
421		✗	static int try_push_frames(AVFilterContext *ctx)
422			{
423		✗	SpectrumSynthContext *s = ctx->priv;
424			int ret, x;
425
426		✗	if (!(s->magnitude && s->phase))
427		✗	return 0;
428
429		✗	switch (s->sliding) {
430		✗	case REPLACE:
431		✗	ret = try_push_frame(ctx, s->xpos);
432		✗	s->xpos++;
433		✗	if (s->xpos >= s->xend)
434		✗	s->xpos = 0;
435		✗	break;
436		✗	case SCROLL:
437		✗	s->xpos = s->xend - 1;
438		✗	ret = try_push_frame(ctx, s->xpos);
439		✗	break;
440		✗	case RSCROLL:
441		✗	s->xpos = 0;
442		✗	ret = try_push_frame(ctx, s->xpos);
443		✗	break;
444		✗	case FULLFRAME:
445		✗	for (x = 0; x < s->xend; x++) {
446		✗	ret = try_push_frame(ctx, x);
447		✗	if (ret < 0)
448		✗	break;
449			}
450		✗	break;
451		✗	default:
452		✗	av_assert0(0);
453			}
454
455		✗	av_frame_free(&s->magnitude);
456		✗	av_frame_free(&s->phase);
457		✗	return ret;
458			}
459
460		✗	static int activate(AVFilterContext *ctx)
461			{
462		✗	SpectrumSynthContext *s = ctx->priv;
463		✗	AVFrame **staging[2] = { &s->magnitude, &s->phase };
464			int64_t pts;
465			int i, ret;
466
467		✗	FF_FILTER_FORWARD_STATUS_BACK_ALL(ctx->outputs[0], ctx);
468
469		✗	for (i = 0; i < 2; i++) {
470		✗	if (*staging[i])
471		✗	continue;
472		✗	ret = ff_inlink_consume_frame(ctx->inputs[i], staging[i]);
473		✗	if (ret < 0)
474		✗	return ret;
475		✗	if (ret) {
476		✗	ff_filter_set_ready(ctx, 10);
477		✗	return try_push_frames(ctx);
478			}
479			}
480
481		✗	for (i = 0; i < 2; i++) {
482		✗	if (ff_inlink_acknowledge_status(ctx->inputs[i], &ret, &pts)) {
483		✗	ff_outlink_set_status(ctx->outputs[0], ret, pts);
484		✗	ff_inlink_set_status(ctx->inputs[1 - i], ret);
485		✗	return 0;
486			}
487			}
488
489		✗	if (ff_outlink_frame_wanted(ctx->outputs[0])) {
490		✗	for (i = 0; i < 2; i++) {
491		✗	if (!*staging[i])
492		✗	ff_inlink_request_frame(ctx->inputs[i]);
493			}
494			}
495
496		✗	return FFERROR_NOT_READY;
497			}
498
499		✗	static av_cold void uninit(AVFilterContext *ctx)
500			{
501		✗	SpectrumSynthContext *s = ctx->priv;
502			int i;
503
504		✗	av_frame_free(&s->magnitude);
505		✗	av_frame_free(&s->phase);
506		✗	av_frame_free(&s->buffer);
507
508		✗	av_tx_uninit(&s->fft);
509
510		✗	if (s->fft_in) {
511		✗	for (i = 0; i < s->channels; i++)
512		✗	av_freep(&s->fft_in[i]);
513			}
514		✗	if (s->fft_out) {
515		✗	for (i = 0; i < s->channels; i++)
516		✗	av_freep(&s->fft_out[i]);
517			}
518		✗	av_freep(&s->fft_in);
519		✗	av_freep(&s->fft_out);
520		✗	av_freep(&s->window_func_lut);
521		✗	}
522
523			static const AVFilterPad spectrumsynth_inputs[] = {
524			{
525			.name = "magnitude",
526			.type = AVMEDIA_TYPE_VIDEO,
527			},
528			{
529			.name = "phase",
530			.type = AVMEDIA_TYPE_VIDEO,
531			},
532			};
533
534			static const AVFilterPad spectrumsynth_outputs[] = {
535			{
536			.name = "default",
537			.type = AVMEDIA_TYPE_AUDIO,
538			.config_props = config_output,
539			},
540			};
541
542			const FFFilter ff_vaf_spectrumsynth = {
543			.p.name = "spectrumsynth",
544			.p.description = NULL_IF_CONFIG_SMALL("Convert input spectrum videos to audio output."),
545			.p.priv_class = &spectrumsynth_class,
546			.uninit = uninit,
547			.activate = activate,
548			.priv_size = sizeof(SpectrumSynthContext),
549			FILTER_INPUTS(spectrumsynth_inputs),
550			FILTER_OUTPUTS(spectrumsynth_outputs),
551			FILTER_QUERY_FUNC2(query_formats),
552			};
553