FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2022 Paul B Mahol
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or
7			* modify it under the terms of the GNU Lesser General Public
8			* License as published by the Free Software Foundation; either
9			* version 2.1 of the License, or (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14			* Lesser General Public License for more details.
15			*
16			* You should have received a copy of the GNU Lesser General Public
17			* License along with FFmpeg; if not, write to the Free Software
18			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19			*/
20
21			#include <float.h>
22			#include <math.h>
23
24			#include "libavutil/mem.h"
25			#include "libavutil/tx.h"
26			#include "libavutil/channel_layout.h"
27			#include "libavutil/float_dsp.h"
28			#include "libavutil/cpu.h"
29			#include "libavutil/opt.h"
30			#include "libavutil/parseutils.h"
31			#include "audio.h"
32			#include "formats.h"
33			#include "video.h"
34			#include "avfilter.h"
35			#include "filters.h"
36			#include "internal.h"
37
38			enum FrequencyScale {
39			FSCALE_LINEAR,
40			FSCALE_LOG,
41			FSCALE_BARK,
42			FSCALE_MEL,
43			FSCALE_ERBS,
44			FSCALE_SQRT,
45			FSCALE_CBRT,
46			FSCALE_QDRT,
47			FSCALE_FM,
48			NB_FSCALE
49			};
50
51			enum IntensityScale {
52			ISCALE_LOG,
53			ISCALE_LINEAR,
54			ISCALE_SQRT,
55			ISCALE_CBRT,
56			ISCALE_QDRT,
57			NB_ISCALE
58			};
59
60			enum DirectionMode {
61			DIRECTION_LR,
62			DIRECTION_RL,
63			DIRECTION_UD,
64			DIRECTION_DU,
65			NB_DIRECTION
66			};
67
68			enum SlideMode {
69			SLIDE_REPLACE,
70			SLIDE_SCROLL,
71			SLIDE_FRAME,
72			NB_SLIDE
73			};
74
75			typedef struct ShowCWTContext {
76			const AVClass *class;
77			int w, h;
78			int mode;
79			char *rate_str;
80			AVRational auto_frame_rate;
81			AVRational frame_rate;
82			AVTXContext **fft, **ifft;
83			av_tx_fn tx_fn, itx_fn;
84			int fft_size, ifft_size;
85			int pos;
86			int64_t in_pts;
87			int64_t old_pts;
88			int64_t eof_pts;
89			float *frequency_band;
90			AVComplexFloat **kernel;
91			unsigned *index;
92			int *kernel_start, *kernel_stop;
93			AVFrame *cache;
94			AVFrame *outpicref;
95			AVFrame *fft_in;
96			AVFrame *fft_out;
97			AVFrame *dst_x;
98			AVFrame *src_x;
99			AVFrame *ifft_in;
100			AVFrame *ifft_out;
101			AVFrame *ch_out;
102			AVFrame *over;
103			AVFrame *bh_out;
104			int nb_threads;
105			int nb_channels;
106			int nb_consumed_samples;
107			int pps;
108			int eof;
109			int slide;
110			int new_frame;
111			int direction;
112			int hop_size, ihop_size;
113			int hop_index, ihop_index;
114			int input_padding_size, output_padding_size;
115			int input_sample_count, output_sample_count;
116			int frequency_band_count;
117			float logarithmic_basis;
118			int intensity_scale;
119			int frequency_scale;
120			float minimum_frequency, maximum_frequency;
121			float minimum_intensity, maximum_intensity;
122			float deviation;
123			float bar_ratio;
124			int bar_size;
125			int sono_size;
126			float rotation;
127
128			AVFloatDSPContext *fdsp;
129			} ShowCWTContext;
130
131			#define OFFSET(x) offsetof(ShowCWTContext, x)
132			#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
133
134			static const AVOption showcwt_options[] = {
135			{ "size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "640x512"}, 0, 0, FLAGS },
136			{ "s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "640x512"}, 0, 0, FLAGS },
137			{ "rate", "set video rate", OFFSET(rate_str), AV_OPT_TYPE_STRING, {.str = "25"}, 0, 0, FLAGS },
138			{ "r", "set video rate", OFFSET(rate_str), AV_OPT_TYPE_STRING, {.str = "25"}, 0, 0, FLAGS },
139			{ "scale", "set frequency scale", OFFSET(frequency_scale), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_FSCALE-1, FLAGS, .unit = "scale" },
140			{ "linear", "linear", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_LINEAR}, 0, 0, FLAGS, .unit = "scale" },
141			{ "log", "logarithmic", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_LOG}, 0, 0, FLAGS, .unit = "scale" },
142			{ "bark", "bark", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_BARK}, 0, 0, FLAGS, .unit = "scale" },
143			{ "mel", "mel", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_MEL}, 0, 0, FLAGS, .unit = "scale" },
144			{ "erbs", "erbs", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_ERBS}, 0, 0, FLAGS, .unit = "scale" },
145			{ "sqrt", "sqrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_SQRT}, 0, 0, FLAGS, .unit = "scale" },
146			{ "cbrt", "cbrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_CBRT}, 0, 0, FLAGS, .unit = "scale" },
147			{ "qdrt", "qdrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_QDRT}, 0, 0, FLAGS, .unit = "scale" },
148			{ "fm", "fm", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_FM}, 0, 0, FLAGS, .unit = "scale" },
149			{ "iscale", "set intensity scale", OFFSET(intensity_scale),AV_OPT_TYPE_INT, {.i64=0}, 0, NB_ISCALE-1, FLAGS, .unit = "iscale" },
150			{ "linear", "linear", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_LINEAR}, 0, 0, FLAGS, .unit = "iscale" },
151			{ "log", "logarithmic", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_LOG}, 0, 0, FLAGS, .unit = "iscale" },
152			{ "sqrt", "sqrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_SQRT}, 0, 0, FLAGS, .unit = "iscale" },
153			{ "cbrt", "cbrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_CBRT}, 0, 0, FLAGS, .unit = "iscale" },
154			{ "qdrt", "qdrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_QDRT}, 0, 0, FLAGS, .unit = "iscale" },
155			{ "min", "set minimum frequency", OFFSET(minimum_frequency), AV_OPT_TYPE_FLOAT, {.dbl = 20.}, 1, 192000, FLAGS },
156			{ "max", "set maximum frequency", OFFSET(maximum_frequency), AV_OPT_TYPE_FLOAT, {.dbl = 20000.}, 1, 192000, FLAGS },
157			{ "imin", "set minimum intensity", OFFSET(minimum_intensity), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
158			{ "imax", "set maximum intensity", OFFSET(maximum_intensity), AV_OPT_TYPE_FLOAT, {.dbl = 1.}, 0, 1, FLAGS },
159			{ "logb", "set logarithmic basis", OFFSET(logarithmic_basis), AV_OPT_TYPE_FLOAT, {.dbl = 0.0001}, 0, 1, FLAGS },
160			{ "deviation", "set frequency deviation", OFFSET(deviation), AV_OPT_TYPE_FLOAT, {.dbl = 1.}, 0, 100, FLAGS },
161			{ "pps", "set pixels per second", OFFSET(pps), AV_OPT_TYPE_INT, {.i64 = 64}, 1, 1024, FLAGS },
162			{ "mode", "set output mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=0}, 0, 4, FLAGS, .unit = "mode" },
163			{ "magnitude", "magnitude", 0, AV_OPT_TYPE_CONST,{.i64=0}, 0, 0, FLAGS, .unit = "mode" },
164			{ "phase", "phase", 0, AV_OPT_TYPE_CONST,{.i64=1}, 0, 0, FLAGS, .unit = "mode" },
165			{ "magphase", "magnitude+phase", 0, AV_OPT_TYPE_CONST,{.i64=2}, 0, 0, FLAGS, .unit = "mode" },
166			{ "channel", "color per channel", 0, AV_OPT_TYPE_CONST,{.i64=3}, 0, 0, FLAGS, .unit = "mode" },
167			{ "stereo", "stereo difference", 0, AV_OPT_TYPE_CONST,{.i64=4}, 0, 0, FLAGS, .unit = "mode" },
168			{ "slide", "set slide mode", OFFSET(slide), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_SLIDE-1, FLAGS, .unit = "slide" },
169			{ "replace", "replace", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_REPLACE},0, 0, FLAGS, .unit = "slide" },
170			{ "scroll", "scroll", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_SCROLL}, 0, 0, FLAGS, .unit = "slide" },
171			{ "frame", "frame", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_FRAME}, 0, 0, FLAGS, .unit = "slide" },
172			{ "direction", "set direction mode", OFFSET(direction), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_DIRECTION-1, FLAGS, .unit = "direction" },
173			{ "lr", "left to right", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_LR}, 0, 0, FLAGS, .unit = "direction" },
174			{ "rl", "right to left", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_RL}, 0, 0, FLAGS, .unit = "direction" },
175			{ "ud", "up to down", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_UD}, 0, 0, FLAGS, .unit = "direction" },
176			{ "du", "down to up", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_DU}, 0, 0, FLAGS, .unit = "direction" },
177			{ "bar", "set bargraph ratio", OFFSET(bar_ratio), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
178			{ "rotation", "set color rotation", OFFSET(rotation), AV_OPT_TYPE_FLOAT, {.dbl = 0}, -1, 1, FLAGS },
179			{ NULL }
180			};
181
182			AVFILTER_DEFINE_CLASS(showcwt);
183
184		✗	static av_cold void uninit(AVFilterContext *ctx)
185			{
186		✗	ShowCWTContext *s = ctx->priv;
187
188		✗	av_freep(&s->frequency_band);
189		✗	av_freep(&s->kernel_start);
190		✗	av_freep(&s->kernel_stop);
191		✗	av_freep(&s->index);
192
193		✗	av_frame_free(&s->cache);
194		✗	av_frame_free(&s->outpicref);
195		✗	av_frame_free(&s->fft_in);
196		✗	av_frame_free(&s->fft_out);
197		✗	av_frame_free(&s->dst_x);
198		✗	av_frame_free(&s->src_x);
199		✗	av_frame_free(&s->ifft_in);
200		✗	av_frame_free(&s->ifft_out);
201		✗	av_frame_free(&s->ch_out);
202		✗	av_frame_free(&s->over);
203		✗	av_frame_free(&s->bh_out);
204
205		✗	if (s->fft) {
206		✗	for (int n = 0; n < s->nb_threads; n++)
207		✗	av_tx_uninit(&s->fft[n]);
208		✗	av_freep(&s->fft);
209			}
210
211		✗	if (s->ifft) {
212		✗	for (int n = 0; n < s->nb_threads; n++)
213		✗	av_tx_uninit(&s->ifft[n]);
214		✗	av_freep(&s->ifft);
215			}
216
217		✗	if (s->kernel) {
218		✗	for (int n = 0; n < s->frequency_band_count; n++)
219		✗	av_freep(&s->kernel[n]);
220			}
221		✗	av_freep(&s->kernel);
222
223		✗	av_freep(&s->fdsp);
224		✗	}
225
226		✗	static int query_formats(AVFilterContext *ctx)
227			{
228		✗	AVFilterFormats *formats = NULL;
229		✗	AVFilterChannelLayouts *layouts = NULL;
230		✗	AVFilterLink *inlink = ctx->inputs[0];
231		✗	AVFilterLink *outlink = ctx->outputs[0];
232			static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE };
233			static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE };
234			int ret;
235
236		✗	formats = ff_make_format_list(sample_fmts);
237		✗	if ((ret = ff_formats_ref(formats, &inlink->outcfg.formats)) < 0)
238		✗	return ret;
239
240		✗	layouts = ff_all_channel_counts();
241		✗	if ((ret = ff_channel_layouts_ref(layouts, &inlink->outcfg.channel_layouts)) < 0)
242		✗	return ret;
243
244		✗	formats = ff_all_samplerates();
245		✗	if ((ret = ff_formats_ref(formats, &inlink->outcfg.samplerates)) < 0)
246		✗	return ret;
247
248		✗	formats = ff_make_format_list(pix_fmts);
249		✗	if ((ret = ff_formats_ref(formats, &outlink->incfg.formats)) < 0)
250		✗	return ret;
251
252		✗	return 0;
253			}
254
255		✗	static float frequency_band(float *frequency_band,
256			int frequency_band_count,
257			float frequency_range,
258			float frequency_offset,
259			int frequency_scale, float deviation)
260			{
261		✗	float ret = 0.f;
262
263		✗	deviation = sqrtf(deviation / (4.f * M_PI)); // Heisenberg Gabor Limit
264		✗	for (int y = 0; y < frequency_band_count; y++) {
265		✗	float frequency = frequency_range * (1.f - (float)y / frequency_band_count) + frequency_offset;
266		✗	float frequency_derivative = frequency_range / frequency_band_count;
267
268		✗	switch (frequency_scale) {
269		✗	case FSCALE_LOG:
270		✗	frequency = powf(2.f, frequency);
271		✗	frequency_derivative *= logf(2.f) * frequency;
272		✗	break;
273		✗	case FSCALE_BARK:
274		✗	frequency = 600.f * sinhf(frequency / 6.f);
275		✗	frequency_derivative *= sqrtf(frequency * frequency + 360000.f) / 6.f;
276		✗	break;
277		✗	case FSCALE_MEL:
278		✗	frequency = 700.f * (powf(10.f, frequency / 2595.f) - 1.f);
279		✗	frequency_derivative *= (frequency + 700.f) * logf(10.f) / 2595.f;
280		✗	break;
281		✗	case FSCALE_ERBS:
282		✗	frequency = 676170.4f / (47.06538f - expf(frequency * 0.08950404f)) - 14678.49f;
283		✗	frequency_derivative *= (frequency * frequency + 14990.4f * frequency + 4577850.f) / 160514.f;
284		✗	break;
285		✗	case FSCALE_SQRT:
286		✗	frequency = frequency * frequency;
287		✗	frequency_derivative *= 2.f * sqrtf(frequency);
288		✗	break;
289		✗	case FSCALE_CBRT:
290		✗	frequency = frequency * frequency * frequency;
291		✗	frequency_derivative *= 3.f * powf(frequency, 2.f / 3.f);
292		✗	break;
293		✗	case FSCALE_QDRT:
294		✗	frequency = frequency * frequency * frequency * frequency;
295		✗	frequency_derivative *= 4.f * powf(frequency, 3.f / 4.f);
296		✗	break;
297		✗	case FSCALE_FM:
298		✗	frequency = 2.f * powf(frequency, 3.f / 2.f) / 3.f;
299		✗	frequency_derivative *= sqrtf(frequency);
300		✗	break;
301			}
302
303		✗	frequency_band[y*2 ] = frequency;
304		✗	frequency_band[y*2+1] = frequency_derivative * deviation;
305
306		✗	ret = 1.f / (frequency_derivative * deviation);
307			}
308
309		✗	return ret;
310			}
311
312		✗	static float remap_log(ShowCWTContext *s, float value, int iscale, float log_factor)
313			{
314		✗	const float max = s->maximum_intensity;
315		✗	const float min = s->minimum_intensity;
316			float ret;
317
318		✗	value += min;
319
320		✗	switch (iscale) {
321		✗	case ISCALE_LINEAR:
322		✗	ret = max - expf(value / log_factor);
323		✗	break;
324		✗	case ISCALE_LOG:
325		✗	value = logf(value) * log_factor;
326		✗	ret = max - av_clipf(value, 0.f, 1.f);
327		✗	break;
328		✗	case ISCALE_SQRT:
329		✗	value = max - expf(value / log_factor);
330		✗	ret = sqrtf(value);
331		✗	break;
332		✗	case ISCALE_CBRT:
333		✗	value = max - expf(value / log_factor);
334		✗	ret = cbrtf(value);
335		✗	break;
336		✗	case ISCALE_QDRT:
337		✗	value = max - expf(value / log_factor);
338		✗	ret = powf(value, 0.25f);
339		✗	break;
340			}
341
342		✗	return av_clipf(ret, 0.f, 1.f);
343			}
344
345		✗	static int run_channel_cwt_prepare(AVFilterContext *ctx, void *arg, int jobnr, int ch)
346			{
347		✗	ShowCWTContext *s = ctx->priv;
348		✗	const int hop_size = s->hop_size;
349		✗	AVFrame *fin = arg;
350		✗	float *cache = (float *)s->cache->extended_data[ch];
351		✗	AVComplexFloat *src = (AVComplexFloat *)s->fft_in->extended_data[ch];
352		✗	AVComplexFloat *dst = (AVComplexFloat *)s->fft_out->extended_data[ch];
353		✗	const int offset = (s->input_padding_size - hop_size) >> 1;
354
355		✗	if (fin) {
356		✗	const float *input = (const float *)fin->extended_data[ch];
357		✗	const int offset = s->hop_size - fin->nb_samples;
358
359		✗	memmove(cache, &cache[fin->nb_samples], offset * sizeof(float));
360		✗	memcpy(&cache[offset], input, fin->nb_samples * sizeof(float));
361			}
362
363		✗	if (fin && s->hop_index + fin->nb_samples < hop_size)
364		✗	return 0;
365
366		✗	memset(src, 0, sizeof(float) * s->fft_size);
367		✗	for (int n = 0; n < hop_size; n++)
368		✗	src[n+offset].re = cache[n];
369
370		✗	s->tx_fn(s->fft[jobnr], dst, src, sizeof(*src));
371
372		✗	return 0;
373			}
374
375			#define DRAW_BAR_COLOR(x) \
376			do { \
377			if (Y <= ht) { \
378			dstY[x] = 0; \
379			dstU[x] = 128; \
380			dstV[x] = 128; \
381			} else { \
382			float mul = (Y - ht) * bh[0]; \
383			dstY[x] = av_clip_uint8(lrintf(Y * mul * 255.f)); \
384			dstU[x] = av_clip_uint8(lrintf((U-0.5f) * 128.f + 128)); \
385			dstV[x] = av_clip_uint8(lrintf((V-0.5f) * 128.f + 128)); \
386			} \
387			} while (0)
388
389		✗	static void draw_bar(ShowCWTContext *s, int y,
390			float Y, float U, float V)
391			{
392		✗	float *bh = ((float *)s->bh_out->extended_data[0]) + y;
393		✗	const ptrdiff_t ylinesize = s->outpicref->linesize[0];
394		✗	const ptrdiff_t ulinesize = s->outpicref->linesize[1];
395		✗	const ptrdiff_t vlinesize = s->outpicref->linesize[2];
396		✗	const int direction = s->direction;
397		✗	const int sono_size = s->sono_size;
398		✗	const int bar_size = s->bar_size;
399		✗	const float rcp_bar_h = 1.f / bar_size;
400			uint8_t *dstY, *dstU, *dstV;
401		✗	const int w = s->w;
402
403		✗	bh[0] = 1.f / (Y + 0.0001f);
404		✗	switch (direction) {
405		✗	case DIRECTION_LR:
406		✗	dstY = s->outpicref->data[0] + y * ylinesize;
407		✗	dstU = s->outpicref->data[1] + y * ulinesize;
408		✗	dstV = s->outpicref->data[2] + y * vlinesize;
409		✗	for (int x = 0; x < bar_size; x++) {
410		✗	float ht = (bar_size - x) * rcp_bar_h;
411		✗	DRAW_BAR_COLOR(x);
412			}
413		✗	break;
414		✗	case DIRECTION_RL:
415		✗	dstY = s->outpicref->data[0] + y * ylinesize;
416		✗	dstU = s->outpicref->data[1] + y * ulinesize;
417		✗	dstV = s->outpicref->data[2] + y * vlinesize;
418		✗	for (int x = 0; x < bar_size; x++) {
419		✗	float ht = x * rcp_bar_h;
420		✗	DRAW_BAR_COLOR(w - bar_size + x);
421			}
422		✗	break;
423		✗	case DIRECTION_UD:
424		✗	dstY = s->outpicref->data[0] + w - 1 - y;
425		✗	dstU = s->outpicref->data[1] + w - 1 - y;
426		✗	dstV = s->outpicref->data[2] + w - 1 - y;
427		✗	for (int x = 0; x < bar_size; x++) {
428		✗	float ht = (bar_size - x) * rcp_bar_h;
429		✗	DRAW_BAR_COLOR(0);
430		✗	dstY += ylinesize;
431		✗	dstU += ulinesize;
432		✗	dstV += vlinesize;
433			}
434		✗	break;
435		✗	case DIRECTION_DU:
436		✗	dstY = s->outpicref->data[0] + w - 1 - y + ylinesize * sono_size;
437		✗	dstU = s->outpicref->data[1] + w - 1 - y + ulinesize * sono_size;
438		✗	dstV = s->outpicref->data[2] + w - 1 - y + vlinesize * sono_size;
439		✗	for (int x = 0; x < bar_size; x++) {
440		✗	float ht = x * rcp_bar_h;
441		✗	DRAW_BAR_COLOR(0);
442		✗	dstY += ylinesize;
443		✗	dstU += ulinesize;
444		✗	dstV += vlinesize;
445			}
446		✗	break;
447			}
448		✗	}
449
450		✗	static int draw(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
451			{
452		✗	ShowCWTContext *s = ctx->priv;
453		✗	const ptrdiff_t ylinesize = s->outpicref->linesize[0];
454		✗	const ptrdiff_t ulinesize = s->outpicref->linesize[1];
455		✗	const ptrdiff_t vlinesize = s->outpicref->linesize[2];
456		✗	const ptrdiff_t alinesize = s->outpicref->linesize[3];
457		✗	const float log_factor = 1.f/logf(s->logarithmic_basis);
458		✗	const int count = s->frequency_band_count;
459		✗	const int start = (count * jobnr) / nb_jobs;
460		✗	const int end = (count * (jobnr+1)) / nb_jobs;
461		✗	const int nb_channels = s->nb_channels;
462		✗	const int iscale = s->intensity_scale;
463		✗	const int ihop_index = s->ihop_index;
464		✗	const int ihop_size = s->ihop_size;
465		✗	const float rotation = s->rotation;
466		✗	const int direction = s->direction;
467			uint8_t *dstY, *dstU, *dstV, *dstA;
468		✗	const int sono_size = s->sono_size;
469		✗	const int bar_size = s->bar_size;
470		✗	const int mode = s->mode;
471		✗	const int w_1 = s->w - 1;
472		✗	const int x = s->pos;
473			float Y, U, V;
474
475		✗	for (int y = start; y < end; y++) {
476		✗	const AVComplexFloat *src = ((const AVComplexFloat *)s->ch_out->extended_data[y]) +
477		✗	0 * ihop_size + ihop_index;
478
479		✗	if (sono_size <= 0)
480		✗	goto skip;
481
482		✗	switch (direction) {
483		✗	case DIRECTION_LR:
484			case DIRECTION_RL:
485		✗	dstY = s->outpicref->data[0] + y * ylinesize;
486		✗	dstU = s->outpicref->data[1] + y * ulinesize;
487		✗	dstV = s->outpicref->data[2] + y * vlinesize;
488		✗	dstA = s->outpicref->data[3] ? s->outpicref->data[3] + y * alinesize : NULL;
489		✗	break;
490		✗	case DIRECTION_UD:
491			case DIRECTION_DU:
492		✗	dstY = s->outpicref->data[0] + x * ylinesize + w_1 - y;
493		✗	dstU = s->outpicref->data[1] + x * ulinesize + w_1 - y;
494		✗	dstV = s->outpicref->data[2] + x * vlinesize + w_1 - y;
495		✗	dstA = s->outpicref->data[3] ? s->outpicref->data[3] + x * alinesize + w_1 - y : NULL;
496		✗	break;
497			}
498
499		✗	switch (s->slide) {
500		✗	case SLIDE_REPLACE:
501			case SLIDE_FRAME:
502			/* nothing to do here */
503		✗	break;
504		✗	case SLIDE_SCROLL:
505		✗	switch (s->direction) {
506		✗	case DIRECTION_RL:
507		✗	memmove(dstY, dstY + 1, w_1);
508		✗	memmove(dstU, dstU + 1, w_1);
509		✗	memmove(dstV, dstV + 1, w_1);
510		✗	if (dstA != NULL)
511		✗	memmove(dstA, dstA + 1, w_1);
512		✗	break;
513		✗	case DIRECTION_LR:
514		✗	memmove(dstY + 1, dstY, w_1);
515		✗	memmove(dstU + 1, dstU, w_1);
516		✗	memmove(dstV + 1, dstV, w_1);
517		✗	if (dstA != NULL)
518		✗	memmove(dstA + 1, dstA, w_1);
519		✗	break;
520			}
521		✗	break;
522			}
523
524		✗	if (direction == DIRECTION_RL ||
525			direction == DIRECTION_LR) {
526		✗	dstY += x;
527		✗	dstU += x;
528		✗	dstV += x;
529		✗	if (dstA != NULL)
530		✗	dstA += x;
531			}
532		✗	skip:
533
534		✗	switch (mode) {
535		✗	case 4:
536			{
537		✗	const AVComplexFloat *src2 = (nb_channels > 1) ? src + ihop_size: src;
538			float z, u, v;
539
540		✗	z = hypotf(src[0].re + src2[0].re, src[0].im + src2[0].im);
541		✗	u = hypotf(src[0].re, src[0].im);
542		✗	v = hypotf(src2[0].re, src2[0].im);
543
544		✗	z = remap_log(s, z, iscale, log_factor);
545		✗	u = remap_log(s, u, iscale, log_factor);
546		✗	v = remap_log(s, v, iscale, log_factor);
547
548		✗	Y = z;
549		✗	U = sinf((v - u) * M_PI_2);
550		✗	V = sinf((u - v) * M_PI_2);
551
552		✗	u = U * cosf(rotation * M_PI) - V * sinf(rotation * M_PI);
553		✗	v = U * sinf(rotation * M_PI) + V * cosf(rotation * M_PI);
554
555		✗	U = 0.5f + 0.5f * z * u;
556		✗	V = 0.5f + 0.5f * z * v;
557
558		✗	if (sono_size > 0) {
559		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
560		✗	dstU[0] = av_clip_uint8(lrintf(U * 255.f));
561		✗	dstV[0] = av_clip_uint8(lrintf(V * 255.f));
562		✗	if (dstA)
563		✗	dstA[0] = dstY[0];
564			}
565
566		✗	if (bar_size > 0)
567		✗	draw_bar(s, y, Y, U, V);
568			}
569		✗	break;
570		✗	case 3:
571			{
572		✗	const int nb_channels = s->nb_channels;
573		✗	const float yf = 1.f / nb_channels;
574
575		✗	Y = 0.f;
576		✗	U = V = 0.5f;
577		✗	for (int ch = 0; ch < nb_channels; ch++) {
578		✗	const AVComplexFloat *srcn = src + ihop_size * ch;
579			float z;
580
581		✗	z = hypotf(srcn[0].re, srcn[0].im);
582		✗	z = remap_log(s, z, iscale, log_factor);
583
584		✗	Y += z * yf;
585		✗	U += z * yf * sinf(2.f * M_PI * (ch * yf + rotation));
586		✗	V += z * yf * cosf(2.f * M_PI * (ch * yf + rotation));
587			}
588
589		✗	if (sono_size > 0) {
590		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
591		✗	dstU[0] = av_clip_uint8(lrintf(U * 255.f));
592		✗	dstV[0] = av_clip_uint8(lrintf(V * 255.f));
593		✗	if (dstA)
594		✗	dstA[0] = dstY[0];
595			}
596
597		✗	if (bar_size > 0)
598		✗	draw_bar(s, y, Y, U, V);
599			}
600		✗	break;
601		✗	case 2:
602		✗	Y = hypotf(src[0].re, src[0].im);
603		✗	Y = remap_log(s, Y, iscale, log_factor);
604		✗	U = atan2f(src[0].im, src[0].re);
605		✗	U = 0.5f + 0.5f * U * Y / M_PI;
606		✗	V = 1.f - U;
607
608		✗	if (sono_size > 0) {
609		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
610		✗	dstU[0] = av_clip_uint8(lrintf(U * 255.f));
611		✗	dstV[0] = av_clip_uint8(lrintf(V * 255.f));
612		✗	if (dstA)
613		✗	dstA[0] = dstY[0];
614			}
615
616		✗	if (bar_size > 0)
617		✗	draw_bar(s, y, Y, U, V);
618		✗	break;
619		✗	case 1:
620		✗	Y = atan2f(src[0].im, src[0].re);
621		✗	Y = 0.5f + 0.5f * Y / M_PI;
622
623		✗	if (sono_size > 0) {
624		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
625		✗	if (dstA)
626		✗	dstA[0] = dstY[0];
627			}
628
629		✗	if (bar_size > 0)
630		✗	draw_bar(s, y, Y, 0.5f, 0.5f);
631		✗	break;
632		✗	case 0:
633		✗	Y = hypotf(src[0].re, src[0].im);
634		✗	Y = remap_log(s, Y, iscale, log_factor);
635
636		✗	if (sono_size > 0) {
637		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
638		✗	if (dstA)
639		✗	dstA[0] = dstY[0];
640			}
641
642		✗	if (bar_size > 0)
643		✗	draw_bar(s, y, Y, 0.5f, 0.5f);
644		✗	break;
645			}
646			}
647
648		✗	return 0;
649			}
650
651		✗	static int run_channel_cwt(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
652			{
653		✗	ShowCWTContext *s = ctx->priv;
654		✗	const int ch = *(int *)arg;
655		✗	const AVComplexFloat *fft_out = (const AVComplexFloat *)s->fft_out->extended_data[ch];
656		✗	AVComplexFloat *isrc = (AVComplexFloat *)s->ifft_in->extended_data[jobnr];
657		✗	AVComplexFloat *idst = (AVComplexFloat *)s->ifft_out->extended_data[jobnr];
658		✗	const int output_padding_size = s->output_padding_size;
659		✗	const int input_padding_size = s->input_padding_size;
660		✗	const float scale = 1.f / input_padding_size;
661		✗	const int ihop_size = s->ihop_size;
662		✗	const int count = s->frequency_band_count;
663		✗	const int start = (count * jobnr) / nb_jobs;
664		✗	const int end = (count * (jobnr+1)) / nb_jobs;
665
666		✗	for (int y = start; y < end; y++) {
667		✗	AVComplexFloat *chout = ((AVComplexFloat *)s->ch_out->extended_data[y]) + ch * ihop_size;
668		✗	AVComplexFloat *over = ((AVComplexFloat *)s->over->extended_data[ch]) + y * ihop_size;
669		✗	AVComplexFloat *dstx = (AVComplexFloat *)s->dst_x->extended_data[jobnr];
670		✗	AVComplexFloat *srcx = (AVComplexFloat *)s->src_x->extended_data[jobnr];
671		✗	const AVComplexFloat *kernel = s->kernel[y];
672		✗	const unsigned *index = (const unsigned *)s->index;
673		✗	const int kernel_start = s->kernel_start[y];
674		✗	const int kernel_stop = s->kernel_stop[y];
675		✗	const int kernel_range = kernel_stop - kernel_start + 1;
676			int offset;
677
678		✗	if (kernel_start >= 0) {
679		✗	offset = 0;
680		✗	memcpy(srcx, fft_out + kernel_start, sizeof(*fft_out) * kernel_range);
681			} else {
682		✗	offset = -kernel_start;
683		✗	memcpy(srcx+offset, fft_out, sizeof(*fft_out) * (kernel_range-offset));
684		✗	memcpy(srcx, fft_out+input_padding_size-offset, sizeof(*fft_out)*offset);
685			}
686
687		✗	s->fdsp->vector_fmul_scalar((float *)srcx, (const float *)srcx, scale, FFALIGN(kernel_range * 2, 4));
688		✗	s->fdsp->vector_fmul((float *)dstx, (const float *)srcx,
689		✗	(const float *)kernel, FFALIGN(kernel_range * 2, 16));
690
691		✗	memset(isrc, 0, sizeof(*isrc) * output_padding_size);
692		✗	if (offset == 0) {
693		✗	const unsigned *kindex = index + kernel_start;
694		✗	for (int i = 0; i < kernel_range; i++) {
695		✗	const unsigned n = kindex[i];
696
697		✗	isrc[n].re += dstx[i].re;
698		✗	isrc[n].im += dstx[i].im;
699			}
700			} else {
701		✗	for (int i = 0; i < kernel_range; i++) {
702		✗	const unsigned n = (i-kernel_start) & (output_padding_size-1);
703
704		✗	isrc[n].re += dstx[i].re;
705		✗	isrc[n].im += dstx[i].im;
706			}
707			}
708
709		✗	s->itx_fn(s->ifft[jobnr], idst, isrc, sizeof(*isrc));
710
711		✗	memcpy(chout, idst, sizeof(*chout) * ihop_size);
712		✗	for (int n = 0; n < ihop_size; n++) {
713		✗	chout[n].re += over[n].re;
714		✗	chout[n].im += over[n].im;
715			}
716		✗	memcpy(over, idst + ihop_size, sizeof(*over) * ihop_size);
717			}
718
719		✗	return 0;
720			}
721
722		✗	static int compute_kernel(AVFilterContext *ctx)
723			{
724		✗	ShowCWTContext *s = ctx->priv;
725		✗	const int size = s->input_padding_size;
726		✗	const int output_sample_count = s->output_sample_count;
727		✗	const int fsize = s->frequency_band_count;
728		✗	int *kernel_start = s->kernel_start;
729		✗	int *kernel_stop = s->kernel_stop;
730		✗	unsigned *index = s->index;
731		✗	int range_min = INT_MAX;
732		✗	int range_max = 0, ret = 0;
733			float *tkernel;
734
735		✗	tkernel = av_malloc_array(size, sizeof(*tkernel));
736		✗	if (!tkernel)
737		✗	return AVERROR(ENOMEM);
738
739		✗	for (int y = 0; y < fsize; y++) {
740		✗	AVComplexFloat *kernel = s->kernel[y];
741		✗	int start = INT_MIN, stop = INT_MAX;
742		✗	const float frequency = s->frequency_band[y*2];
743		✗	const float deviation = 1.f / (s->frequency_band[y*2+1] *
744			output_sample_count);
745		✗	const int a = FFMAX(frequency-12.f*sqrtf(1.f/deviation)-0.5f, -size);
746		✗	const int b = FFMIN(frequency+12.f*sqrtf(1.f/deviation)-0.5f, size+a);
747		✗	const int range = -a;
748
749		✗	memset(tkernel, 0, size * sizeof(*tkernel));
750		✗	for (int n = a; n < b; n++) {
751		✗	float ff, f = n+0.5f-frequency;
752
753		✗	ff = expf(-f*f*deviation);
754		✗	tkernel[n+range] = ff;
755			}
756
757		✗	for (int n = a; n < b; n++) {
758		✗	if (tkernel[n+range] != 0.f) {
759		✗	if (tkernel[n+range] > FLT_MIN)
760		✗	av_log(ctx, AV_LOG_DEBUG, "out of range kernel %g\n", tkernel[n+range]);
761		✗	start = n;
762		✗	break;
763			}
764			}
765
766		✗	for (int n = b; n >= a; n--) {
767		✗	if (tkernel[n+range] != 0.f) {
768		✗	if (tkernel[n+range] > FLT_MIN)
769		✗	av_log(ctx, AV_LOG_DEBUG, "out of range kernel %g\n", tkernel[n+range]);
770		✗	stop = n;
771		✗	break;
772			}
773			}
774
775		✗	if (start == INT_MIN || stop == INT_MAX) {
776		✗	ret = AVERROR(EINVAL);
777		✗	break;
778			}
779
780		✗	kernel_start[y] = start;
781		✗	kernel_stop[y] = stop;
782
783		✗	kernel = av_calloc(FFALIGN(stop-start+1, 16), sizeof(*kernel));
784		✗	if (!kernel) {
785		✗	ret = AVERROR(ENOMEM);
786		✗	break;
787			}
788
789		✗	for (int n = 0; n <= stop - start; n++) {
790		✗	kernel[n].re = tkernel[n+range+start];
791		✗	kernel[n].im = tkernel[n+range+start];
792			}
793
794		✗	range_min = FFMIN(range_min, stop+1-start);
795		✗	range_max = FFMAX(range_max, stop+1-start);
796
797		✗	s->kernel[y] = kernel;
798			}
799
800		✗	for (int n = 0; n < size; n++)
801		✗	index[n] = n & (s->output_padding_size - 1);
802
803		✗	av_log(ctx, AV_LOG_DEBUG, "range_min: %d\n", range_min);
804		✗	av_log(ctx, AV_LOG_DEBUG, "range_max: %d\n", range_max);
805
806		✗	av_freep(&tkernel);
807
808		✗	return ret;
809			}
810
811		✗	static int config_output(AVFilterLink *outlink)
812			{
813		✗	AVFilterContext *ctx = outlink->src;
814		✗	AVFilterLink *inlink = ctx->inputs[0];
815		✗	ShowCWTContext *s = ctx->priv;
816		✗	const float limit_frequency = inlink->sample_rate * 0.5f;
817		✗	float maximum_frequency = fminf(s->maximum_frequency, limit_frequency);
818		✗	float minimum_frequency = s->minimum_frequency;
819		✗	float scale = 1.f, factor;
820			int ret;
821
822		✗	if (minimum_frequency >= maximum_frequency) {
823		✗	av_log(ctx, AV_LOG_ERROR, "min frequency (%f) >= (%f) max frequency\n",
824			minimum_frequency, maximum_frequency);
825		✗	return AVERROR(EINVAL);
826			}
827
828		✗	uninit(ctx);
829
830		✗	s->fdsp = avpriv_float_dsp_alloc(0);
831		✗	if (!s->fdsp)
832		✗	return AVERROR(ENOMEM);
833
834		✗	switch (s->direction) {
835		✗	case DIRECTION_LR:
836			case DIRECTION_RL:
837		✗	s->bar_size = s->w * s->bar_ratio;
838		✗	s->sono_size = s->w - s->bar_size;
839		✗	s->frequency_band_count = s->h;
840		✗	break;
841		✗	case DIRECTION_UD:
842			case DIRECTION_DU:
843		✗	s->bar_size = s->h * s->bar_ratio;
844		✗	s->sono_size = s->h - s->bar_size;
845		✗	s->frequency_band_count = s->w;
846		✗	break;
847			}
848
849		✗	switch (s->frequency_scale) {
850		✗	case FSCALE_LOG:
851		✗	minimum_frequency = logf(minimum_frequency) / logf(2.f);
852		✗	maximum_frequency = logf(maximum_frequency) / logf(2.f);
853		✗	break;
854		✗	case FSCALE_BARK:
855		✗	minimum_frequency = 6.f * asinhf(minimum_frequency / 600.f);
856		✗	maximum_frequency = 6.f * asinhf(maximum_frequency / 600.f);
857		✗	break;
858		✗	case FSCALE_MEL:
859		✗	minimum_frequency = 2595.f * log10f(1.f + minimum_frequency / 700.f);
860		✗	maximum_frequency = 2595.f * log10f(1.f + maximum_frequency / 700.f);
861		✗	break;
862		✗	case FSCALE_ERBS:
863		✗	minimum_frequency = 11.17268f * logf(1.f + (46.06538f * minimum_frequency) / (minimum_frequency + 14678.49f));
864		✗	maximum_frequency = 11.17268f * logf(1.f + (46.06538f * maximum_frequency) / (maximum_frequency + 14678.49f));
865		✗	break;
866		✗	case FSCALE_SQRT:
867		✗	minimum_frequency = sqrtf(minimum_frequency);
868		✗	maximum_frequency = sqrtf(maximum_frequency);
869		✗	break;
870		✗	case FSCALE_CBRT:
871		✗	minimum_frequency = cbrtf(minimum_frequency);
872		✗	maximum_frequency = cbrtf(maximum_frequency);
873		✗	break;
874		✗	case FSCALE_QDRT:
875		✗	minimum_frequency = powf(minimum_frequency, 0.25f);
876		✗	maximum_frequency = powf(maximum_frequency, 0.25f);
877		✗	break;
878		✗	case FSCALE_FM:
879		✗	minimum_frequency = powf(9.f * (minimum_frequency * minimum_frequency) / 4.f, 1.f / 3.f);
880		✗	maximum_frequency = powf(9.f * (maximum_frequency * maximum_frequency) / 4.f, 1.f / 3.f);
881		✗	break;
882			}
883
884		✗	s->frequency_band = av_calloc(s->frequency_band_count,
885			sizeof(*s->frequency_band) * 2);
886		✗	if (!s->frequency_band)
887		✗	return AVERROR(ENOMEM);
888
889		✗	s->nb_consumed_samples = inlink->sample_rate *
890		✗	frequency_band(s->frequency_band,
891			s->frequency_band_count, maximum_frequency - minimum_frequency,
892			minimum_frequency, s->frequency_scale, s->deviation);
893		✗	s->nb_consumed_samples = FFMIN(s->nb_consumed_samples, 65536);
894
895		✗	s->nb_threads = FFMIN(s->frequency_band_count, ff_filter_get_nb_threads(ctx));
896		✗	s->nb_channels = inlink->ch_layout.nb_channels;
897		✗	s->old_pts = AV_NOPTS_VALUE;
898		✗	s->eof_pts = AV_NOPTS_VALUE;
899
900		✗	s->input_sample_count = 1 << (32 - ff_clz(s->nb_consumed_samples));
901		✗	s->input_padding_size = 1 << (32 - ff_clz(s->input_sample_count));
902		✗	s->output_sample_count = FFMAX(1, av_rescale(s->input_sample_count, s->pps, inlink->sample_rate));
903		✗	s->output_padding_size = 1 << (32 - ff_clz(s->output_sample_count));
904
905		✗	s->hop_size = s->input_sample_count;
906		✗	s->ihop_size = s->output_padding_size >> 1;
907
908		✗	outlink->w = s->w;
909		✗	outlink->h = s->h;
910		✗	outlink->sample_aspect_ratio = (AVRational){1,1};
911
912		✗	s->fft_size = FFALIGN(s->input_padding_size, av_cpu_max_align());
913		✗	s->ifft_size = FFALIGN(s->output_padding_size, av_cpu_max_align());
914
915		✗	s->fft = av_calloc(s->nb_threads, sizeof(*s->fft));
916		✗	if (!s->fft)
917		✗	return AVERROR(ENOMEM);
918
919		✗	for (int n = 0; n < s->nb_threads; n++) {
920		✗	ret = av_tx_init(&s->fft[n], &s->tx_fn, AV_TX_FLOAT_FFT, 0, s->input_padding_size, &scale, 0);
921		✗	if (ret < 0)
922		✗	return ret;
923			}
924
925		✗	s->ifft = av_calloc(s->nb_threads, sizeof(*s->ifft));
926		✗	if (!s->ifft)
927		✗	return AVERROR(ENOMEM);
928
929		✗	for (int n = 0; n < s->nb_threads; n++) {
930		✗	ret = av_tx_init(&s->ifft[n], &s->itx_fn, AV_TX_FLOAT_FFT, 1, s->output_padding_size, &scale, 0);
931		✗	if (ret < 0)
932		✗	return ret;
933			}
934
935		✗	s->outpicref = ff_get_video_buffer(outlink, outlink->w, outlink->h);
936		✗	s->fft_in = ff_get_audio_buffer(inlink, s->fft_size * 2);
937		✗	s->fft_out = ff_get_audio_buffer(inlink, s->fft_size * 2);
938		✗	s->dst_x = av_frame_alloc();
939		✗	s->src_x = av_frame_alloc();
940		✗	s->kernel = av_calloc(s->frequency_band_count, sizeof(*s->kernel));
941		✗	s->cache = ff_get_audio_buffer(inlink, s->hop_size);
942		✗	s->over = ff_get_audio_buffer(inlink, s->frequency_band_count * 2 * s->ihop_size);
943		✗	s->bh_out = ff_get_audio_buffer(inlink, s->frequency_band_count);
944		✗	s->ifft_in = av_frame_alloc();
945		✗	s->ifft_out = av_frame_alloc();
946		✗	s->ch_out = av_frame_alloc();
947		✗	s->index = av_calloc(s->input_padding_size, sizeof(*s->index));
948		✗	s->kernel_start = av_calloc(s->frequency_band_count, sizeof(*s->kernel_start));
949		✗	s->kernel_stop = av_calloc(s->frequency_band_count, sizeof(*s->kernel_stop));
950		✗	if (!s->outpicref || !s->fft_in || !s->fft_out || !s->src_x || !s->dst_x || !s->over ||
951		✗	!s->ifft_in || !s->ifft_out || !s->kernel_start || !s->kernel_stop || !s->ch_out ||
952		✗	!s->cache || !s->index || !s->bh_out || !s->kernel)
953		✗	return AVERROR(ENOMEM);
954
955		✗	s->ch_out->format = inlink->format;
956		✗	s->ch_out->nb_samples = 2 * s->ihop_size * inlink->ch_layout.nb_channels;
957		✗	s->ch_out->ch_layout.nb_channels = s->frequency_band_count;
958		✗	ret = av_frame_get_buffer(s->ch_out, 0);
959		✗	if (ret < 0)
960		✗	return ret;
961
962		✗	s->ifft_in->format = inlink->format;
963		✗	s->ifft_in->nb_samples = s->ifft_size * 2;
964		✗	s->ifft_in->ch_layout.nb_channels = s->nb_threads;
965		✗	ret = av_frame_get_buffer(s->ifft_in, 0);
966		✗	if (ret < 0)
967		✗	return ret;
968
969		✗	s->ifft_out->format = inlink->format;
970		✗	s->ifft_out->nb_samples = s->ifft_size * 2;
971		✗	s->ifft_out->ch_layout.nb_channels = s->nb_threads;
972		✗	ret = av_frame_get_buffer(s->ifft_out, 0);
973		✗	if (ret < 0)
974		✗	return ret;
975
976		✗	s->src_x->format = inlink->format;
977		✗	s->src_x->nb_samples = s->fft_size * 2;
978		✗	s->src_x->ch_layout.nb_channels = s->nb_threads;
979		✗	ret = av_frame_get_buffer(s->src_x, 0);
980		✗	if (ret < 0)
981		✗	return ret;
982
983		✗	s->dst_x->format = inlink->format;
984		✗	s->dst_x->nb_samples = s->fft_size * 2;
985		✗	s->dst_x->ch_layout.nb_channels = s->nb_threads;
986		✗	ret = av_frame_get_buffer(s->dst_x, 0);
987		✗	if (ret < 0)
988		✗	return ret;
989
990		✗	s->outpicref->sample_aspect_ratio = (AVRational){1,1};
991
992		✗	for (int y = 0; y < outlink->h; y++) {
993		✗	memset(s->outpicref->data[0] + y * s->outpicref->linesize[0], 0, outlink->w);
994		✗	memset(s->outpicref->data[1] + y * s->outpicref->linesize[1], 128, outlink->w);
995		✗	memset(s->outpicref->data[2] + y * s->outpicref->linesize[2], 128, outlink->w);
996		✗	if (s->outpicref->data[3])
997		✗	memset(s->outpicref->data[3] + y * s->outpicref->linesize[3], 0, outlink->w);
998			}
999
1000		✗	s->outpicref->color_range = AVCOL_RANGE_JPEG;
1001
1002		✗	factor = s->input_padding_size / (float)inlink->sample_rate;
1003		✗	for (int n = 0; n < s->frequency_band_count; n++) {
1004		✗	s->frequency_band[2*n ] *= factor;
1005		✗	s->frequency_band[2*n+1] *= factor;
1006			}
1007
1008		✗	av_log(ctx, AV_LOG_DEBUG, "factor: %f\n", factor);
1009		✗	av_log(ctx, AV_LOG_DEBUG, "nb_consumed_samples: %d\n", s->nb_consumed_samples);
1010		✗	av_log(ctx, AV_LOG_DEBUG, "hop_size: %d\n", s->hop_size);
1011		✗	av_log(ctx, AV_LOG_DEBUG, "ihop_size: %d\n", s->ihop_size);
1012		✗	av_log(ctx, AV_LOG_DEBUG, "input_sample_count: %d\n", s->input_sample_count);
1013		✗	av_log(ctx, AV_LOG_DEBUG, "input_padding_size: %d\n", s->input_padding_size);
1014		✗	av_log(ctx, AV_LOG_DEBUG, "output_sample_count: %d\n", s->output_sample_count);
1015		✗	av_log(ctx, AV_LOG_DEBUG, "output_padding_size: %d\n", s->output_padding_size);
1016
1017		✗	switch (s->direction) {
1018		✗	case DIRECTION_LR:
1019			case DIRECTION_UD:
1020		✗	s->pos = s->bar_size;
1021		✗	break;
1022		✗	case DIRECTION_RL:
1023			case DIRECTION_DU:
1024		✗	s->pos = s->sono_size;
1025		✗	break;
1026			}
1027
1028		✗	s->auto_frame_rate = av_make_q(inlink->sample_rate, s->hop_size);
1029		✗	if (strcmp(s->rate_str, "auto")) {
1030		✗	ret = av_parse_video_rate(&s->frame_rate, s->rate_str);
1031		✗	if (ret < 0)
1032		✗	return ret;
1033			} else {
1034		✗	s->frame_rate = s->auto_frame_rate;
1035			}
1036		✗	outlink->frame_rate = s->frame_rate;
1037		✗	outlink->time_base = av_inv_q(outlink->frame_rate);
1038
1039		✗	ret = compute_kernel(ctx);
1040		✗	if (ret < 0)
1041		✗	return ret;
1042
1043		✗	return 0;
1044			}
1045
1046		✗	static int output_frame(AVFilterContext *ctx)
1047			{
1048		✗	AVFilterLink *outlink = ctx->outputs[0];
1049		✗	AVFilterLink *inlink = ctx->inputs[0];
1050		✗	ShowCWTContext *s = ctx->priv;
1051		✗	const int nb_planes = 3 + (s->outpicref->data[3] != NULL);
1052			int ret;
1053
1054		✗	switch (s->slide) {
1055		✗	case SLIDE_SCROLL:
1056		✗	switch (s->direction) {
1057		✗	case DIRECTION_UD:
1058		✗	for (int p = 0; p < nb_planes; p++) {
1059		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1060
1061		✗	for (int y = s->h - 1; y > s->bar_size; y--) {
1062		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1063
1064		✗	memmove(dst, dst - linesize, s->w);
1065			}
1066			}
1067		✗	break;
1068		✗	case DIRECTION_DU:
1069		✗	for (int p = 0; p < nb_planes; p++) {
1070		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1071
1072		✗	for (int y = 0; y < s->sono_size; y++) {
1073		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1074
1075		✗	memmove(dst, dst + linesize, s->w);
1076			}
1077			}
1078		✗	break;
1079			}
1080		✗	break;
1081			}
1082
1083		✗	ff_filter_execute(ctx, draw, NULL, NULL, s->nb_threads);
1084
1085		✗	switch (s->slide) {
1086		✗	case SLIDE_REPLACE:
1087			case SLIDE_FRAME:
1088		✗	switch (s->direction) {
1089		✗	case DIRECTION_LR:
1090		✗	s->pos++;
1091		✗	if (s->pos >= s->w) {
1092		✗	s->pos = s->bar_size;
1093		✗	s->new_frame = 1;
1094			}
1095		✗	break;
1096		✗	case DIRECTION_RL:
1097		✗	s->pos--;
1098		✗	if (s->pos < 0) {
1099		✗	s->pos = s->sono_size;
1100		✗	s->new_frame = 1;
1101			}
1102		✗	break;
1103		✗	case DIRECTION_UD:
1104		✗	s->pos++;
1105		✗	if (s->pos >= s->h) {
1106		✗	s->pos = s->bar_size;
1107		✗	s->new_frame = 1;
1108			}
1109		✗	break;
1110		✗	case DIRECTION_DU:
1111		✗	s->pos--;
1112		✗	if (s->pos < 0) {
1113		✗	s->pos = s->sono_size;
1114		✗	s->new_frame = 1;
1115			}
1116		✗	break;
1117			}
1118		✗	break;
1119		✗	case SLIDE_SCROLL:
1120		✗	switch (s->direction) {
1121		✗	case DIRECTION_UD:
1122			case DIRECTION_LR:
1123		✗	s->pos = s->bar_size;
1124		✗	break;
1125		✗	case DIRECTION_RL:
1126			case DIRECTION_DU:
1127		✗	s->pos = s->sono_size;
1128		✗	break;
1129			}
1130		✗	break;
1131			}
1132
1133		✗	if (s->slide == SLIDE_FRAME && s->eof) {
1134		✗	switch (s->direction) {
1135		✗	case DIRECTION_LR:
1136		✗	for (int p = 0; p < nb_planes; p++) {
1137		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1138		✗	const int size = s->w - s->pos;
1139		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1140		✗	const int x = s->pos;
1141
1142		✗	for (int y = 0; y < s->h; y++) {
1143		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize + x;
1144
1145		✗	memset(dst, fill, size);
1146			}
1147			}
1148		✗	break;
1149		✗	case DIRECTION_RL:
1150		✗	for (int p = 0; p < nb_planes; p++) {
1151		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1152		✗	const int size = s->w - s->pos;
1153		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1154
1155		✗	for (int y = 0; y < s->h; y++) {
1156		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1157
1158		✗	memset(dst, fill, size);
1159			}
1160			}
1161		✗	break;
1162		✗	case DIRECTION_UD:
1163		✗	for (int p = 0; p < nb_planes; p++) {
1164		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1165		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1166
1167		✗	for (int y = s->pos; y < s->h; y++) {
1168		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1169
1170		✗	memset(dst, fill, s->w);
1171			}
1172			}
1173		✗	break;
1174		✗	case DIRECTION_DU:
1175		✗	for (int p = 0; p < nb_planes; p++) {
1176		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1177		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1178
1179		✗	for (int y = s->h - s->pos; y >= 0; y--) {
1180		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1181
1182		✗	memset(dst, fill, s->w);
1183			}
1184			}
1185		✗	break;
1186			}
1187			}
1188
1189		✗	s->new_frame = s->slide == SLIDE_FRAME && (s->new_frame || s->eof);
1190
1191		✗	if (s->slide != SLIDE_FRAME || s->new_frame == 1) {
1192		✗	int64_t pts_offset = s->new_frame ? 0LL : av_rescale(s->ihop_index, s->hop_size, s->ihop_size);
1193		✗	const int offset = (s->input_padding_size - s->hop_size) >> 1;
1194
1195		✗	pts_offset = av_rescale_q(pts_offset - offset, av_make_q(1, inlink->sample_rate), inlink->time_base);
1196		✗	s->outpicref->pts = av_rescale_q(s->in_pts + pts_offset, inlink->time_base, outlink->time_base);
1197		✗	s->outpicref->duration = 1;
1198			}
1199
1200		✗	s->ihop_index++;
1201		✗	if (s->ihop_index >= s->ihop_size)
1202		✗	s->ihop_index = s->hop_index = 0;
1203
1204		✗	if (s->slide == SLIDE_FRAME && s->new_frame == 0)
1205		✗	return 1;
1206
1207		✗	if (s->old_pts < s->outpicref->pts) {
1208		✗	AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
1209		✗	if (!out)
1210		✗	return AVERROR(ENOMEM);
1211		✗	ret = av_frame_copy_props(out, s->outpicref);
1212		✗	if (ret < 0)
1213		✗	goto fail;
1214		✗	ret = av_frame_copy(out, s->outpicref);
1215		✗	if (ret < 0)
1216		✗	goto fail;
1217		✗	s->old_pts = s->outpicref->pts;
1218		✗	s->new_frame = 0;
1219		✗	ret = ff_filter_frame(outlink, out);
1220		✗	if (ret <= 0)
1221		✗	return ret;
1222		✗	fail:
1223		✗	av_frame_free(&out);
1224		✗	return ret;
1225			}
1226
1227		✗	return 1;
1228			}
1229
1230		✗	static int run_channels_cwt_prepare(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
1231			{
1232		✗	ShowCWTContext *s = ctx->priv;
1233		✗	const int count = s->nb_channels;
1234		✗	const int start = (count * jobnr) / nb_jobs;
1235		✗	const int end = (count * (jobnr+1)) / nb_jobs;
1236
1237		✗	for (int ch = start; ch < end; ch++)
1238		✗	run_channel_cwt_prepare(ctx, arg, jobnr, ch);
1239
1240		✗	return 0;
1241			}
1242
1243		✗	static int activate(AVFilterContext *ctx)
1244			{
1245		✗	AVFilterLink *inlink = ctx->inputs[0];
1246		✗	AVFilterLink *outlink = ctx->outputs[0];
1247		✗	ShowCWTContext *s = ctx->priv;
1248		✗	int ret = 0, status;
1249			int64_t pts;
1250
1251		✗	FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
1252
1253		✗	if (s->outpicref) {
1254		✗	AVFrame *fin = NULL;
1255
1256		✗	if (s->hop_index < s->hop_size) {
1257		✗	if (!s->eof) {
1258		✗	ret = ff_inlink_consume_samples(inlink, 1, s->hop_size - s->hop_index, &fin);
1259		✗	if (ret < 0)
1260		✗	return ret;
1261			}
1262
1263		✗	if (ret > 0 || s->eof) {
1264		✗	ff_filter_execute(ctx, run_channels_cwt_prepare, fin, NULL,
1265		✗	FFMIN(s->nb_threads, s->nb_channels));
1266		✗	if (fin) {
1267		✗	if (s->hop_index == 0) {
1268		✗	s->in_pts = fin->pts;
1269		✗	if (s->old_pts == AV_NOPTS_VALUE)
1270		✗	s->old_pts = av_rescale_q(s->in_pts, inlink->time_base, outlink->time_base) - 1;
1271			}
1272		✗	s->hop_index += fin->nb_samples;
1273		✗	av_frame_free(&fin);
1274			} else {
1275		✗	s->hop_index = s->hop_size;
1276			}
1277			}
1278			}
1279
1280		✗	if (s->hop_index >= s->hop_size || s->ihop_index > 0) {
1281		✗	for (int ch = 0; ch < s->nb_channels && s->ihop_index == 0; ch++) {
1282		✗	ff_filter_execute(ctx, run_channel_cwt, (void *)&ch, NULL,
1283			s->nb_threads);
1284			}
1285
1286		✗	ret = output_frame(ctx);
1287		✗	if (ret != 1)
1288		✗	return ret;
1289			}
1290			}
1291
1292		✗	if (s->eof) {
1293		✗	if (s->slide == SLIDE_FRAME)
1294		✗	ret = output_frame(ctx);
1295		✗	ff_outlink_set_status(outlink, AVERROR_EOF, s->eof_pts);
1296		✗	return ret;
1297			}
1298
1299		✗	if (!s->eof && ff_inlink_acknowledge_status(inlink, &status, &pts)) {
1300		✗	if (status == AVERROR_EOF) {
1301		✗	s->eof = 1;
1302		✗	ff_filter_set_ready(ctx, 10);
1303		✗	s->eof_pts = av_rescale_q(pts, inlink->time_base, outlink->time_base);
1304		✗	return 0;
1305			}
1306			}
1307
1308		✗	if (ff_inlink_queued_samples(inlink) > 0 || s->ihop_index ||
1309		✗	s->hop_index >= s->hop_size || s->eof) {
1310		✗	ff_filter_set_ready(ctx, 10);
1311		✗	return 0;
1312			}
1313
1314		✗	if (ff_outlink_frame_wanted(outlink)) {
1315		✗	ff_inlink_request_frame(inlink);
1316		✗	return 0;
1317			}
1318
1319		✗	return FFERROR_NOT_READY;
1320			}
1321
1322			static const AVFilterPad showcwt_outputs[] = {
1323			{
1324			.name = "default",
1325			.type = AVMEDIA_TYPE_VIDEO,
1326			.config_props = config_output,
1327			},
1328			};
1329
1330			const AVFilter ff_avf_showcwt = {
1331			.name = "showcwt",
1332			.description = NULL_IF_CONFIG_SMALL("Convert input audio to a CWT (Continuous Wavelet Transform) spectrum video output."),
1333			.uninit = uninit,
1334			.priv_size = sizeof(ShowCWTContext),
1335			FILTER_INPUTS(ff_audio_default_filterpad),
1336			FILTER_OUTPUTS(showcwt_outputs),
1337			FILTER_QUERY_FUNC(query_formats),
1338			.activate = activate,
1339			.priv_class = &showcwt_class,
1340			.flags = AVFILTER_FLAG_SLICE_THREADS,
1341			};
1342