FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2022 Paul B Mahol
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or
7			* modify it under the terms of the GNU Lesser General Public
8			* License as published by the Free Software Foundation; either
9			* version 2.1 of the License, or (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14			* Lesser General Public License for more details.
15			*
16			* You should have received a copy of the GNU Lesser General Public
17			* License along with FFmpeg; if not, write to the Free Software
18			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19			*/
20
21			#include <float.h>
22			#include <math.h>
23
24			#include "libavutil/mem.h"
25			#include "libavutil/tx.h"
26			#include "libavutil/channel_layout.h"
27			#include "libavutil/float_dsp.h"
28			#include "libavutil/cpu.h"
29			#include "libavutil/opt.h"
30			#include "libavutil/parseutils.h"
31			#include "audio.h"
32			#include "formats.h"
33			#include "video.h"
34			#include "avfilter.h"
35			#include "filters.h"
36
37			enum FrequencyScale {
38			FSCALE_LINEAR,
39			FSCALE_LOG,
40			FSCALE_BARK,
41			FSCALE_MEL,
42			FSCALE_ERBS,
43			FSCALE_SQRT,
44			FSCALE_CBRT,
45			FSCALE_QDRT,
46			FSCALE_FM,
47			NB_FSCALE
48			};
49
50			enum IntensityScale {
51			ISCALE_LOG,
52			ISCALE_LINEAR,
53			ISCALE_SQRT,
54			ISCALE_CBRT,
55			ISCALE_QDRT,
56			NB_ISCALE
57			};
58
59			enum DirectionMode {
60			DIRECTION_LR,
61			DIRECTION_RL,
62			DIRECTION_UD,
63			DIRECTION_DU,
64			NB_DIRECTION
65			};
66
67			enum SlideMode {
68			SLIDE_REPLACE,
69			SLIDE_SCROLL,
70			SLIDE_FRAME,
71			NB_SLIDE
72			};
73
74			typedef struct ShowCWTContext {
75			const AVClass *class;
76			int w, h;
77			int mode;
78			char *rate_str;
79			AVRational auto_frame_rate;
80			AVRational frame_rate;
81			AVTXContext **fft, **ifft;
82			av_tx_fn tx_fn, itx_fn;
83			int fft_size, ifft_size;
84			int pos;
85			int64_t in_pts;
86			int64_t old_pts;
87			int64_t eof_pts;
88			float *frequency_band;
89			AVComplexFloat **kernel;
90			unsigned *index;
91			int *kernel_start, *kernel_stop;
92			AVFrame *cache;
93			AVFrame *outpicref;
94			AVFrame *fft_in;
95			AVFrame *fft_out;
96			AVFrame *dst_x;
97			AVFrame *src_x;
98			AVFrame *ifft_in;
99			AVFrame *ifft_out;
100			AVFrame *ch_out;
101			AVFrame *over;
102			AVFrame *bh_out;
103			int nb_threads;
104			int nb_channels;
105			int nb_consumed_samples;
106			int pps;
107			int eof;
108			int slide;
109			int new_frame;
110			int direction;
111			int hop_size, ihop_size;
112			int hop_index, ihop_index;
113			int input_padding_size, output_padding_size;
114			int input_sample_count, output_sample_count;
115			int frequency_band_count;
116			float logarithmic_basis;
117			int intensity_scale;
118			int frequency_scale;
119			float minimum_frequency, maximum_frequency;
120			float minimum_intensity, maximum_intensity;
121			float deviation;
122			float bar_ratio;
123			int bar_size;
124			int sono_size;
125			float rotation;
126
127			AVFloatDSPContext *fdsp;
128			} ShowCWTContext;
129
130			#define OFFSET(x) offsetof(ShowCWTContext, x)
131			#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
132
133			static const AVOption showcwt_options[] = {
134			{ "size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "640x512"}, 0, 0, FLAGS },
135			{ "s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "640x512"}, 0, 0, FLAGS },
136			{ "rate", "set video rate", OFFSET(rate_str), AV_OPT_TYPE_STRING, {.str = "25"}, 0, 0, FLAGS },
137			{ "r", "set video rate", OFFSET(rate_str), AV_OPT_TYPE_STRING, {.str = "25"}, 0, 0, FLAGS },
138			{ "scale", "set frequency scale", OFFSET(frequency_scale), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_FSCALE-1, FLAGS, .unit = "scale" },
139			{ "linear", "linear", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_LINEAR}, 0, 0, FLAGS, .unit = "scale" },
140			{ "log", "logarithmic", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_LOG}, 0, 0, FLAGS, .unit = "scale" },
141			{ "bark", "bark", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_BARK}, 0, 0, FLAGS, .unit = "scale" },
142			{ "mel", "mel", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_MEL}, 0, 0, FLAGS, .unit = "scale" },
143			{ "erbs", "erbs", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_ERBS}, 0, 0, FLAGS, .unit = "scale" },
144			{ "sqrt", "sqrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_SQRT}, 0, 0, FLAGS, .unit = "scale" },
145			{ "cbrt", "cbrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_CBRT}, 0, 0, FLAGS, .unit = "scale" },
146			{ "qdrt", "qdrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_QDRT}, 0, 0, FLAGS, .unit = "scale" },
147			{ "fm", "fm", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_FM}, 0, 0, FLAGS, .unit = "scale" },
148			{ "iscale", "set intensity scale", OFFSET(intensity_scale),AV_OPT_TYPE_INT, {.i64=0}, 0, NB_ISCALE-1, FLAGS, .unit = "iscale" },
149			{ "linear", "linear", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_LINEAR}, 0, 0, FLAGS, .unit = "iscale" },
150			{ "log", "logarithmic", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_LOG}, 0, 0, FLAGS, .unit = "iscale" },
151			{ "sqrt", "sqrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_SQRT}, 0, 0, FLAGS, .unit = "iscale" },
152			{ "cbrt", "cbrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_CBRT}, 0, 0, FLAGS, .unit = "iscale" },
153			{ "qdrt", "qdrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_QDRT}, 0, 0, FLAGS, .unit = "iscale" },
154			{ "min", "set minimum frequency", OFFSET(minimum_frequency), AV_OPT_TYPE_FLOAT, {.dbl = 20.}, 1, 192000, FLAGS },
155			{ "max", "set maximum frequency", OFFSET(maximum_frequency), AV_OPT_TYPE_FLOAT, {.dbl = 20000.}, 1, 192000, FLAGS },
156			{ "imin", "set minimum intensity", OFFSET(minimum_intensity), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
157			{ "imax", "set maximum intensity", OFFSET(maximum_intensity), AV_OPT_TYPE_FLOAT, {.dbl = 1.}, 0, 1, FLAGS },
158			{ "logb", "set logarithmic basis", OFFSET(logarithmic_basis), AV_OPT_TYPE_FLOAT, {.dbl = 0.0001}, 0, 1, FLAGS },
159			{ "deviation", "set frequency deviation", OFFSET(deviation), AV_OPT_TYPE_FLOAT, {.dbl = 1.}, 0, 100, FLAGS },
160			{ "pps", "set pixels per second", OFFSET(pps), AV_OPT_TYPE_INT, {.i64 = 64}, 1, 1024, FLAGS },
161			{ "mode", "set output mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=0}, 0, 4, FLAGS, .unit = "mode" },
162			{ "magnitude", "magnitude", 0, AV_OPT_TYPE_CONST,{.i64=0}, 0, 0, FLAGS, .unit = "mode" },
163			{ "phase", "phase", 0, AV_OPT_TYPE_CONST,{.i64=1}, 0, 0, FLAGS, .unit = "mode" },
164			{ "magphase", "magnitude+phase", 0, AV_OPT_TYPE_CONST,{.i64=2}, 0, 0, FLAGS, .unit = "mode" },
165			{ "channel", "color per channel", 0, AV_OPT_TYPE_CONST,{.i64=3}, 0, 0, FLAGS, .unit = "mode" },
166			{ "stereo", "stereo difference", 0, AV_OPT_TYPE_CONST,{.i64=4}, 0, 0, FLAGS, .unit = "mode" },
167			{ "slide", "set slide mode", OFFSET(slide), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_SLIDE-1, FLAGS, .unit = "slide" },
168			{ "replace", "replace", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_REPLACE},0, 0, FLAGS, .unit = "slide" },
169			{ "scroll", "scroll", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_SCROLL}, 0, 0, FLAGS, .unit = "slide" },
170			{ "frame", "frame", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_FRAME}, 0, 0, FLAGS, .unit = "slide" },
171			{ "direction", "set direction mode", OFFSET(direction), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_DIRECTION-1, FLAGS, .unit = "direction" },
172			{ "lr", "left to right", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_LR}, 0, 0, FLAGS, .unit = "direction" },
173			{ "rl", "right to left", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_RL}, 0, 0, FLAGS, .unit = "direction" },
174			{ "ud", "up to down", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_UD}, 0, 0, FLAGS, .unit = "direction" },
175			{ "du", "down to up", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_DU}, 0, 0, FLAGS, .unit = "direction" },
176			{ "bar", "set bargraph ratio", OFFSET(bar_ratio), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
177			{ "rotation", "set color rotation", OFFSET(rotation), AV_OPT_TYPE_FLOAT, {.dbl = 0}, -1, 1, FLAGS },
178			{ NULL }
179			};
180
181			AVFILTER_DEFINE_CLASS(showcwt);
182
183		✗	static av_cold void uninit(AVFilterContext *ctx)
184			{
185		✗	ShowCWTContext *s = ctx->priv;
186
187		✗	av_freep(&s->frequency_band);
188		✗	av_freep(&s->kernel_start);
189		✗	av_freep(&s->kernel_stop);
190		✗	av_freep(&s->index);
191
192		✗	av_frame_free(&s->cache);
193		✗	av_frame_free(&s->outpicref);
194		✗	av_frame_free(&s->fft_in);
195		✗	av_frame_free(&s->fft_out);
196		✗	av_frame_free(&s->dst_x);
197		✗	av_frame_free(&s->src_x);
198		✗	av_frame_free(&s->ifft_in);
199		✗	av_frame_free(&s->ifft_out);
200		✗	av_frame_free(&s->ch_out);
201		✗	av_frame_free(&s->over);
202		✗	av_frame_free(&s->bh_out);
203
204		✗	if (s->fft) {
205		✗	for (int n = 0; n < s->nb_threads; n++)
206		✗	av_tx_uninit(&s->fft[n]);
207		✗	av_freep(&s->fft);
208			}
209
210		✗	if (s->ifft) {
211		✗	for (int n = 0; n < s->nb_threads; n++)
212		✗	av_tx_uninit(&s->ifft[n]);
213		✗	av_freep(&s->ifft);
214			}
215
216		✗	if (s->kernel) {
217		✗	for (int n = 0; n < s->frequency_band_count; n++)
218		✗	av_freep(&s->kernel[n]);
219			}
220		✗	av_freep(&s->kernel);
221
222		✗	av_freep(&s->fdsp);
223		✗	}
224
225		✗	static int query_formats(const AVFilterContext *ctx,
226			AVFilterFormatsConfig **cfg_in,
227			AVFilterFormatsConfig **cfg_out)
228			{
229		✗	AVFilterFormats *formats = NULL;
230			static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE };
231			static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE };
232			int ret;
233
234		✗	formats = ff_make_format_list(sample_fmts);
235		✗	if ((ret = ff_formats_ref(formats, &cfg_in[0]->formats)) < 0)
236		✗	return ret;
237
238		✗	formats = ff_make_format_list(pix_fmts);
239		✗	if ((ret = ff_formats_ref(formats, &cfg_out[0]->formats)) < 0)
240		✗	return ret;
241
242		✗	return 0;
243			}
244
245		✗	static float frequency_band(float *frequency_band,
246			int frequency_band_count,
247			float frequency_range,
248			float frequency_offset,
249			int frequency_scale, float deviation)
250			{
251		✗	float ret = 0.f;
252
253		✗	deviation = sqrtf(deviation / (4.f * M_PI)); // Heisenberg Gabor Limit
254		✗	for (int y = 0; y < frequency_band_count; y++) {
255		✗	float frequency = frequency_range * (1.f - (float)y / frequency_band_count) + frequency_offset;
256		✗	float frequency_derivative = frequency_range / frequency_band_count;
257
258		✗	switch (frequency_scale) {
259		✗	case FSCALE_LOG:
260		✗	frequency = powf(2.f, frequency);
261		✗	frequency_derivative *= logf(2.f) * frequency;
262		✗	break;
263		✗	case FSCALE_BARK:
264		✗	frequency = 600.f * sinhf(frequency / 6.f);
265		✗	frequency_derivative *= sqrtf(frequency * frequency + 360000.f) / 6.f;
266		✗	break;
267		✗	case FSCALE_MEL:
268		✗	frequency = 700.f * (powf(10.f, frequency / 2595.f) - 1.f);
269		✗	frequency_derivative *= (frequency + 700.f) * logf(10.f) / 2595.f;
270		✗	break;
271		✗	case FSCALE_ERBS:
272		✗	frequency = 676170.4f / (47.06538f - expf(frequency * 0.08950404f)) - 14678.49f;
273		✗	frequency_derivative *= (frequency * frequency + 14990.4f * frequency + 4577850.f) / 160514.f;
274		✗	break;
275		✗	case FSCALE_SQRT:
276		✗	frequency = frequency * frequency;
277		✗	frequency_derivative *= 2.f * sqrtf(frequency);
278		✗	break;
279		✗	case FSCALE_CBRT:
280		✗	frequency = frequency * frequency * frequency;
281		✗	frequency_derivative *= 3.f * powf(frequency, 2.f / 3.f);
282		✗	break;
283		✗	case FSCALE_QDRT:
284		✗	frequency = frequency * frequency * frequency * frequency;
285		✗	frequency_derivative *= 4.f * powf(frequency, 3.f / 4.f);
286		✗	break;
287		✗	case FSCALE_FM:
288		✗	frequency = 2.f * powf(frequency, 3.f / 2.f) / 3.f;
289		✗	frequency_derivative *= sqrtf(frequency);
290		✗	break;
291			}
292
293		✗	frequency_band[y*2 ] = frequency;
294		✗	frequency_band[y*2+1] = frequency_derivative * deviation;
295
296		✗	ret = 1.f / (frequency_derivative * deviation);
297			}
298
299		✗	return ret;
300			}
301
302		✗	static float remap_log(ShowCWTContext *s, float value, int iscale, float log_factor)
303			{
304		✗	const float max = s->maximum_intensity;
305		✗	const float min = s->minimum_intensity;
306			float ret;
307
308		✗	value += min;
309
310		✗	switch (iscale) {
311		✗	case ISCALE_LINEAR:
312		✗	ret = max - expf(value / log_factor);
313		✗	break;
314		✗	case ISCALE_LOG:
315		✗	value = logf(value) * log_factor;
316		✗	ret = max - av_clipf(value, 0.f, 1.f);
317		✗	break;
318		✗	case ISCALE_SQRT:
319		✗	value = max - expf(value / log_factor);
320		✗	ret = sqrtf(value);
321		✗	break;
322		✗	case ISCALE_CBRT:
323		✗	value = max - expf(value / log_factor);
324		✗	ret = cbrtf(value);
325		✗	break;
326		✗	case ISCALE_QDRT:
327		✗	value = max - expf(value / log_factor);
328		✗	ret = powf(value, 0.25f);
329		✗	break;
330			}
331
332		✗	return av_clipf(ret, 0.f, 1.f);
333			}
334
335		✗	static int run_channel_cwt_prepare(AVFilterContext *ctx, void *arg, int jobnr, int ch)
336			{
337		✗	ShowCWTContext *s = ctx->priv;
338		✗	const int hop_size = s->hop_size;
339		✗	AVFrame *fin = arg;
340		✗	float *cache = (float *)s->cache->extended_data[ch];
341		✗	AVComplexFloat *src = (AVComplexFloat *)s->fft_in->extended_data[ch];
342		✗	AVComplexFloat *dst = (AVComplexFloat *)s->fft_out->extended_data[ch];
343		✗	const int offset = (s->input_padding_size - hop_size) >> 1;
344
345		✗	if (fin) {
346		✗	const float *input = (const float *)fin->extended_data[ch];
347		✗	const int offset = s->hop_size - fin->nb_samples;
348
349		✗	memmove(cache, &cache[fin->nb_samples], offset * sizeof(float));
350		✗	memcpy(&cache[offset], input, fin->nb_samples * sizeof(float));
351			}
352
353		✗	if (fin && s->hop_index + fin->nb_samples < hop_size)
354		✗	return 0;
355
356		✗	memset(src, 0, sizeof(float) * s->fft_size);
357		✗	for (int n = 0; n < hop_size; n++)
358		✗	src[n+offset].re = cache[n];
359
360		✗	s->tx_fn(s->fft[jobnr], dst, src, sizeof(*src));
361
362		✗	return 0;
363			}
364
365			#define DRAW_BAR_COLOR(x) \
366			do { \
367			if (Y <= ht) { \
368			dstY[x] = 0; \
369			dstU[x] = 128; \
370			dstV[x] = 128; \
371			} else { \
372			float mul = (Y - ht) * bh[0]; \
373			dstY[x] = av_clip_uint8(lrintf(Y * mul * 255.f)); \
374			dstU[x] = av_clip_uint8(lrintf((U-0.5f) * 128.f + 128)); \
375			dstV[x] = av_clip_uint8(lrintf((V-0.5f) * 128.f + 128)); \
376			} \
377			} while (0)
378
379		✗	static void draw_bar(ShowCWTContext *s, int y,
380			float Y, float U, float V)
381			{
382		✗	float *bh = ((float *)s->bh_out->extended_data[0]) + y;
383		✗	const ptrdiff_t ylinesize = s->outpicref->linesize[0];
384		✗	const ptrdiff_t ulinesize = s->outpicref->linesize[1];
385		✗	const ptrdiff_t vlinesize = s->outpicref->linesize[2];
386		✗	const int direction = s->direction;
387		✗	const int sono_size = s->sono_size;
388		✗	const int bar_size = s->bar_size;
389		✗	const float rcp_bar_h = 1.f / bar_size;
390			uint8_t *dstY, *dstU, *dstV;
391		✗	const int w = s->w;
392
393		✗	bh[0] = 1.f / (Y + 0.0001f);
394		✗	switch (direction) {
395		✗	case DIRECTION_LR:
396		✗	dstY = s->outpicref->data[0] + y * ylinesize;
397		✗	dstU = s->outpicref->data[1] + y * ulinesize;
398		✗	dstV = s->outpicref->data[2] + y * vlinesize;
399		✗	for (int x = 0; x < bar_size; x++) {
400		✗	float ht = (bar_size - x) * rcp_bar_h;
401		✗	DRAW_BAR_COLOR(x);
402			}
403		✗	break;
404		✗	case DIRECTION_RL:
405		✗	dstY = s->outpicref->data[0] + y * ylinesize;
406		✗	dstU = s->outpicref->data[1] + y * ulinesize;
407		✗	dstV = s->outpicref->data[2] + y * vlinesize;
408		✗	for (int x = 0; x < bar_size; x++) {
409		✗	float ht = x * rcp_bar_h;
410		✗	DRAW_BAR_COLOR(w - bar_size + x);
411			}
412		✗	break;
413		✗	case DIRECTION_UD:
414		✗	dstY = s->outpicref->data[0] + w - 1 - y;
415		✗	dstU = s->outpicref->data[1] + w - 1 - y;
416		✗	dstV = s->outpicref->data[2] + w - 1 - y;
417		✗	for (int x = 0; x < bar_size; x++) {
418		✗	float ht = (bar_size - x) * rcp_bar_h;
419		✗	DRAW_BAR_COLOR(0);
420		✗	dstY += ylinesize;
421		✗	dstU += ulinesize;
422		✗	dstV += vlinesize;
423			}
424		✗	break;
425		✗	case DIRECTION_DU:
426		✗	dstY = s->outpicref->data[0] + w - 1 - y + ylinesize * sono_size;
427		✗	dstU = s->outpicref->data[1] + w - 1 - y + ulinesize * sono_size;
428		✗	dstV = s->outpicref->data[2] + w - 1 - y + vlinesize * sono_size;
429		✗	for (int x = 0; x < bar_size; x++) {
430		✗	float ht = x * rcp_bar_h;
431		✗	DRAW_BAR_COLOR(0);
432		✗	dstY += ylinesize;
433		✗	dstU += ulinesize;
434		✗	dstV += vlinesize;
435			}
436		✗	break;
437			}
438		✗	}
439
440		✗	static int draw(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
441			{
442		✗	ShowCWTContext *s = ctx->priv;
443		✗	const ptrdiff_t ylinesize = s->outpicref->linesize[0];
444		✗	const ptrdiff_t ulinesize = s->outpicref->linesize[1];
445		✗	const ptrdiff_t vlinesize = s->outpicref->linesize[2];
446		✗	const ptrdiff_t alinesize = s->outpicref->linesize[3];
447		✗	const float log_factor = 1.f/logf(s->logarithmic_basis);
448		✗	const int count = s->frequency_band_count;
449		✗	const int start = (count * jobnr) / nb_jobs;
450		✗	const int end = (count * (jobnr+1)) / nb_jobs;
451		✗	const int nb_channels = s->nb_channels;
452		✗	const int iscale = s->intensity_scale;
453		✗	const int ihop_index = s->ihop_index;
454		✗	const int ihop_size = s->ihop_size;
455		✗	const float rotation = s->rotation;
456		✗	const int direction = s->direction;
457			uint8_t *dstY, *dstU, *dstV, *dstA;
458		✗	const int sono_size = s->sono_size;
459		✗	const int bar_size = s->bar_size;
460		✗	const int mode = s->mode;
461		✗	const int w_1 = s->w - 1;
462		✗	const int x = s->pos;
463			float Y, U, V;
464
465		✗	for (int y = start; y < end; y++) {
466		✗	const AVComplexFloat *src = ((const AVComplexFloat *)s->ch_out->extended_data[y]) +
467		✗	0 * ihop_size + ihop_index;
468
469		✗	if (sono_size <= 0)
470		✗	goto skip;
471
472		✗	switch (direction) {
473		✗	case DIRECTION_LR:
474			case DIRECTION_RL:
475		✗	dstY = s->outpicref->data[0] + y * ylinesize;
476		✗	dstU = s->outpicref->data[1] + y * ulinesize;
477		✗	dstV = s->outpicref->data[2] + y * vlinesize;
478		✗	dstA = s->outpicref->data[3] ? s->outpicref->data[3] + y * alinesize : NULL;
479		✗	break;
480		✗	case DIRECTION_UD:
481			case DIRECTION_DU:
482		✗	dstY = s->outpicref->data[0] + x * ylinesize + w_1 - y;
483		✗	dstU = s->outpicref->data[1] + x * ulinesize + w_1 - y;
484		✗	dstV = s->outpicref->data[2] + x * vlinesize + w_1 - y;
485		✗	dstA = s->outpicref->data[3] ? s->outpicref->data[3] + x * alinesize + w_1 - y : NULL;
486		✗	break;
487			}
488
489		✗	switch (s->slide) {
490		✗	case SLIDE_REPLACE:
491			case SLIDE_FRAME:
492			/* nothing to do here */
493		✗	break;
494		✗	case SLIDE_SCROLL:
495		✗	switch (s->direction) {
496		✗	case DIRECTION_RL:
497		✗	memmove(dstY, dstY + 1, w_1);
498		✗	memmove(dstU, dstU + 1, w_1);
499		✗	memmove(dstV, dstV + 1, w_1);
500		✗	if (dstA != NULL)
501		✗	memmove(dstA, dstA + 1, w_1);
502		✗	break;
503		✗	case DIRECTION_LR:
504		✗	memmove(dstY + 1, dstY, w_1);
505		✗	memmove(dstU + 1, dstU, w_1);
506		✗	memmove(dstV + 1, dstV, w_1);
507		✗	if (dstA != NULL)
508		✗	memmove(dstA + 1, dstA, w_1);
509		✗	break;
510			}
511		✗	break;
512			}
513
514		✗	if (direction == DIRECTION_RL ||
515			direction == DIRECTION_LR) {
516		✗	dstY += x;
517		✗	dstU += x;
518		✗	dstV += x;
519		✗	if (dstA != NULL)
520		✗	dstA += x;
521			}
522		✗	skip:
523
524		✗	switch (mode) {
525		✗	case 4:
526			{
527		✗	const AVComplexFloat *src2 = (nb_channels > 1) ? src + ihop_size: src;
528			float z, u, v;
529
530		✗	z = hypotf(src[0].re + src2[0].re, src[0].im + src2[0].im);
531		✗	u = hypotf(src[0].re, src[0].im);
532		✗	v = hypotf(src2[0].re, src2[0].im);
533
534		✗	z = remap_log(s, z, iscale, log_factor);
535		✗	u = remap_log(s, u, iscale, log_factor);
536		✗	v = remap_log(s, v, iscale, log_factor);
537
538		✗	Y = z;
539		✗	U = sinf((v - u) * M_PI_2);
540		✗	V = sinf((u - v) * M_PI_2);
541
542		✗	u = U * cosf(rotation * M_PI) - V * sinf(rotation * M_PI);
543		✗	v = U * sinf(rotation * M_PI) + V * cosf(rotation * M_PI);
544
545		✗	U = 0.5f + 0.5f * z * u;
546		✗	V = 0.5f + 0.5f * z * v;
547
548		✗	if (sono_size > 0) {
549		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
550		✗	dstU[0] = av_clip_uint8(lrintf(U * 255.f));
551		✗	dstV[0] = av_clip_uint8(lrintf(V * 255.f));
552		✗	if (dstA)
553		✗	dstA[0] = dstY[0];
554			}
555
556		✗	if (bar_size > 0)
557		✗	draw_bar(s, y, Y, U, V);
558			}
559		✗	break;
560		✗	case 3:
561			{
562		✗	const int nb_channels = s->nb_channels;
563		✗	const float yf = 1.f / nb_channels;
564
565		✗	Y = 0.f;
566		✗	U = V = 0.5f;
567		✗	for (int ch = 0; ch < nb_channels; ch++) {
568		✗	const AVComplexFloat *srcn = src + ihop_size * ch;
569			float z;
570
571		✗	z = hypotf(srcn[0].re, srcn[0].im);
572		✗	z = remap_log(s, z, iscale, log_factor);
573
574		✗	Y += z * yf;
575		✗	U += z * yf * sinf(2.f * M_PI * (ch * yf + rotation));
576		✗	V += z * yf * cosf(2.f * M_PI * (ch * yf + rotation));
577			}
578
579		✗	if (sono_size > 0) {
580		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
581		✗	dstU[0] = av_clip_uint8(lrintf(U * 255.f));
582		✗	dstV[0] = av_clip_uint8(lrintf(V * 255.f));
583		✗	if (dstA)
584		✗	dstA[0] = dstY[0];
585			}
586
587		✗	if (bar_size > 0)
588		✗	draw_bar(s, y, Y, U, V);
589			}
590		✗	break;
591		✗	case 2:
592		✗	Y = hypotf(src[0].re, src[0].im);
593		✗	Y = remap_log(s, Y, iscale, log_factor);
594		✗	U = atan2f(src[0].im, src[0].re);
595		✗	U = 0.5f + 0.5f * U * Y / M_PI;
596		✗	V = 1.f - U;
597
598		✗	if (sono_size > 0) {
599		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
600		✗	dstU[0] = av_clip_uint8(lrintf(U * 255.f));
601		✗	dstV[0] = av_clip_uint8(lrintf(V * 255.f));
602		✗	if (dstA)
603		✗	dstA[0] = dstY[0];
604			}
605
606		✗	if (bar_size > 0)
607		✗	draw_bar(s, y, Y, U, V);
608		✗	break;
609		✗	case 1:
610		✗	Y = atan2f(src[0].im, src[0].re);
611		✗	Y = 0.5f + 0.5f * Y / M_PI;
612
613		✗	if (sono_size > 0) {
614		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
615		✗	if (dstA)
616		✗	dstA[0] = dstY[0];
617			}
618
619		✗	if (bar_size > 0)
620		✗	draw_bar(s, y, Y, 0.5f, 0.5f);
621		✗	break;
622		✗	case 0:
623		✗	Y = hypotf(src[0].re, src[0].im);
624		✗	Y = remap_log(s, Y, iscale, log_factor);
625
626		✗	if (sono_size > 0) {
627		✗	dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
628		✗	if (dstA)
629		✗	dstA[0] = dstY[0];
630			}
631
632		✗	if (bar_size > 0)
633		✗	draw_bar(s, y, Y, 0.5f, 0.5f);
634		✗	break;
635			}
636			}
637
638		✗	return 0;
639			}
640
641		✗	static int run_channel_cwt(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
642			{
643		✗	ShowCWTContext *s = ctx->priv;
644		✗	const int ch = *(int *)arg;
645		✗	const AVComplexFloat *fft_out = (const AVComplexFloat *)s->fft_out->extended_data[ch];
646		✗	AVComplexFloat *isrc = (AVComplexFloat *)s->ifft_in->extended_data[jobnr];
647		✗	AVComplexFloat *idst = (AVComplexFloat *)s->ifft_out->extended_data[jobnr];
648		✗	const int output_padding_size = s->output_padding_size;
649		✗	const int input_padding_size = s->input_padding_size;
650		✗	const float scale = 1.f / input_padding_size;
651		✗	const int ihop_size = s->ihop_size;
652		✗	const int count = s->frequency_band_count;
653		✗	const int start = (count * jobnr) / nb_jobs;
654		✗	const int end = (count * (jobnr+1)) / nb_jobs;
655
656		✗	for (int y = start; y < end; y++) {
657		✗	AVComplexFloat *chout = ((AVComplexFloat *)s->ch_out->extended_data[y]) + ch * ihop_size;
658		✗	AVComplexFloat *over = ((AVComplexFloat *)s->over->extended_data[ch]) + y * ihop_size;
659		✗	AVComplexFloat *dstx = (AVComplexFloat *)s->dst_x->extended_data[jobnr];
660		✗	AVComplexFloat *srcx = (AVComplexFloat *)s->src_x->extended_data[jobnr];
661		✗	const AVComplexFloat *kernel = s->kernel[y];
662		✗	const unsigned *index = (const unsigned *)s->index;
663		✗	const int kernel_start = s->kernel_start[y];
664		✗	const int kernel_stop = s->kernel_stop[y];
665		✗	const int kernel_range = kernel_stop - kernel_start + 1;
666			int offset;
667
668		✗	if (kernel_start >= 0) {
669		✗	offset = 0;
670		✗	memcpy(srcx, fft_out + kernel_start, sizeof(*fft_out) * kernel_range);
671			} else {
672		✗	offset = -kernel_start;
673		✗	memcpy(srcx+offset, fft_out, sizeof(*fft_out) * (kernel_range-offset));
674		✗	memcpy(srcx, fft_out+input_padding_size-offset, sizeof(*fft_out)*offset);
675			}
676
677		✗	s->fdsp->vector_fmul_scalar((float *)srcx, (const float *)srcx, scale, FFALIGN(kernel_range * 2, 4));
678		✗	s->fdsp->vector_fmul((float *)dstx, (const float *)srcx,
679		✗	(const float *)kernel, FFALIGN(kernel_range * 2, 16));
680
681		✗	memset(isrc, 0, sizeof(*isrc) * output_padding_size);
682		✗	if (offset == 0) {
683		✗	const unsigned *kindex = index + kernel_start;
684		✗	for (int i = 0; i < kernel_range; i++) {
685		✗	const unsigned n = kindex[i];
686
687		✗	isrc[n].re += dstx[i].re;
688		✗	isrc[n].im += dstx[i].im;
689			}
690			} else {
691		✗	for (int i = 0; i < kernel_range; i++) {
692		✗	const unsigned n = (i-kernel_start) & (output_padding_size-1);
693
694		✗	isrc[n].re += dstx[i].re;
695		✗	isrc[n].im += dstx[i].im;
696			}
697			}
698
699		✗	s->itx_fn(s->ifft[jobnr], idst, isrc, sizeof(*isrc));
700
701		✗	memcpy(chout, idst, sizeof(*chout) * ihop_size);
702		✗	for (int n = 0; n < ihop_size; n++) {
703		✗	chout[n].re += over[n].re;
704		✗	chout[n].im += over[n].im;
705			}
706		✗	memcpy(over, idst + ihop_size, sizeof(*over) * ihop_size);
707			}
708
709		✗	return 0;
710			}
711
712		✗	static int compute_kernel(AVFilterContext *ctx)
713			{
714		✗	ShowCWTContext *s = ctx->priv;
715		✗	const int size = s->input_padding_size;
716		✗	const int output_sample_count = s->output_sample_count;
717		✗	const int fsize = s->frequency_band_count;
718		✗	int *kernel_start = s->kernel_start;
719		✗	int *kernel_stop = s->kernel_stop;
720		✗	unsigned *index = s->index;
721		✗	int range_min = INT_MAX;
722		✗	int range_max = 0, ret = 0;
723			float *tkernel;
724
725		✗	tkernel = av_malloc_array(size, sizeof(*tkernel));
726		✗	if (!tkernel)
727		✗	return AVERROR(ENOMEM);
728
729		✗	for (int y = 0; y < fsize; y++) {
730		✗	AVComplexFloat *kernel = s->kernel[y];
731		✗	int start = INT_MIN, stop = INT_MAX;
732		✗	const float frequency = s->frequency_band[y*2];
733		✗	const float deviation = 1.f / (s->frequency_band[y*2+1] *
734			output_sample_count);
735		✗	const int a = FFMAX(frequency-12.f*sqrtf(1.f/deviation)-0.5f, -size);
736		✗	const int b = FFMIN(frequency+12.f*sqrtf(1.f/deviation)-0.5f, size+a);
737		✗	const int range = -a;
738
739		✗	memset(tkernel, 0, size * sizeof(*tkernel));
740		✗	for (int n = a; n < b; n++) {
741		✗	float ff, f = n+0.5f-frequency;
742
743		✗	ff = expf(-f*f*deviation);
744		✗	tkernel[n+range] = ff;
745			}
746
747		✗	for (int n = a; n < b; n++) {
748		✗	if (tkernel[n+range] != 0.f) {
749		✗	if (tkernel[n+range] > FLT_MIN)
750		✗	av_log(ctx, AV_LOG_DEBUG, "out of range kernel %g\n", tkernel[n+range]);
751		✗	start = n;
752		✗	break;
753			}
754			}
755
756		✗	for (int n = b; n >= a; n--) {
757		✗	if (tkernel[n+range] != 0.f) {
758		✗	if (tkernel[n+range] > FLT_MIN)
759		✗	av_log(ctx, AV_LOG_DEBUG, "out of range kernel %g\n", tkernel[n+range]);
760		✗	stop = n;
761		✗	break;
762			}
763			}
764
765		✗	if (start == INT_MIN || stop == INT_MAX) {
766		✗	ret = AVERROR(EINVAL);
767		✗	break;
768			}
769
770		✗	kernel_start[y] = start;
771		✗	kernel_stop[y] = stop;
772
773		✗	kernel = av_calloc(FFALIGN(stop-start+1, 16), sizeof(*kernel));
774		✗	if (!kernel) {
775		✗	ret = AVERROR(ENOMEM);
776		✗	break;
777			}
778
779		✗	for (int n = 0; n <= stop - start; n++) {
780		✗	kernel[n].re = tkernel[n+range+start];
781		✗	kernel[n].im = tkernel[n+range+start];
782			}
783
784		✗	range_min = FFMIN(range_min, stop+1-start);
785		✗	range_max = FFMAX(range_max, stop+1-start);
786
787		✗	s->kernel[y] = kernel;
788			}
789
790		✗	for (int n = 0; n < size; n++)
791		✗	index[n] = n & (s->output_padding_size - 1);
792
793		✗	av_log(ctx, AV_LOG_DEBUG, "range_min: %d\n", range_min);
794		✗	av_log(ctx, AV_LOG_DEBUG, "range_max: %d\n", range_max);
795
796		✗	av_freep(&tkernel);
797
798		✗	return ret;
799			}
800
801		✗	static int config_output(AVFilterLink *outlink)
802			{
803		✗	FilterLink *l = ff_filter_link(outlink);
804		✗	AVFilterContext *ctx = outlink->src;
805		✗	AVFilterLink *inlink = ctx->inputs[0];
806		✗	ShowCWTContext *s = ctx->priv;
807		✗	const float limit_frequency = inlink->sample_rate * 0.5f;
808		✗	float maximum_frequency = fminf(s->maximum_frequency, limit_frequency);
809		✗	float minimum_frequency = s->minimum_frequency;
810		✗	float scale = 1.f, factor;
811			int ret;
812
813		✗	if (minimum_frequency >= maximum_frequency) {
814		✗	av_log(ctx, AV_LOG_ERROR, "min frequency (%f) >= (%f) max frequency\n",
815			minimum_frequency, maximum_frequency);
816		✗	return AVERROR(EINVAL);
817			}
818
819		✗	uninit(ctx);
820
821		✗	s->fdsp = avpriv_float_dsp_alloc(0);
822		✗	if (!s->fdsp)
823		✗	return AVERROR(ENOMEM);
824
825		✗	switch (s->direction) {
826		✗	case DIRECTION_LR:
827			case DIRECTION_RL:
828		✗	s->bar_size = s->w * s->bar_ratio;
829		✗	s->sono_size = s->w - s->bar_size;
830		✗	s->frequency_band_count = s->h;
831		✗	break;
832		✗	case DIRECTION_UD:
833			case DIRECTION_DU:
834		✗	s->bar_size = s->h * s->bar_ratio;
835		✗	s->sono_size = s->h - s->bar_size;
836		✗	s->frequency_band_count = s->w;
837		✗	break;
838			}
839
840		✗	switch (s->frequency_scale) {
841		✗	case FSCALE_LOG:
842		✗	minimum_frequency = logf(minimum_frequency) / logf(2.f);
843		✗	maximum_frequency = logf(maximum_frequency) / logf(2.f);
844		✗	break;
845		✗	case FSCALE_BARK:
846		✗	minimum_frequency = 6.f * asinhf(minimum_frequency / 600.f);
847		✗	maximum_frequency = 6.f * asinhf(maximum_frequency / 600.f);
848		✗	break;
849		✗	case FSCALE_MEL:
850		✗	minimum_frequency = 2595.f * log10f(1.f + minimum_frequency / 700.f);
851		✗	maximum_frequency = 2595.f * log10f(1.f + maximum_frequency / 700.f);
852		✗	break;
853		✗	case FSCALE_ERBS:
854		✗	minimum_frequency = 11.17268f * logf(1.f + (46.06538f * minimum_frequency) / (minimum_frequency + 14678.49f));
855		✗	maximum_frequency = 11.17268f * logf(1.f + (46.06538f * maximum_frequency) / (maximum_frequency + 14678.49f));
856		✗	break;
857		✗	case FSCALE_SQRT:
858		✗	minimum_frequency = sqrtf(minimum_frequency);
859		✗	maximum_frequency = sqrtf(maximum_frequency);
860		✗	break;
861		✗	case FSCALE_CBRT:
862		✗	minimum_frequency = cbrtf(minimum_frequency);
863		✗	maximum_frequency = cbrtf(maximum_frequency);
864		✗	break;
865		✗	case FSCALE_QDRT:
866		✗	minimum_frequency = powf(minimum_frequency, 0.25f);
867		✗	maximum_frequency = powf(maximum_frequency, 0.25f);
868		✗	break;
869		✗	case FSCALE_FM:
870		✗	minimum_frequency = powf(9.f * (minimum_frequency * minimum_frequency) / 4.f, 1.f / 3.f);
871		✗	maximum_frequency = powf(9.f * (maximum_frequency * maximum_frequency) / 4.f, 1.f / 3.f);
872		✗	break;
873			}
874
875		✗	s->frequency_band = av_calloc(s->frequency_band_count,
876			sizeof(*s->frequency_band) * 2);
877		✗	if (!s->frequency_band)
878		✗	return AVERROR(ENOMEM);
879
880		✗	s->nb_consumed_samples = inlink->sample_rate *
881		✗	frequency_band(s->frequency_band,
882			s->frequency_band_count, maximum_frequency - minimum_frequency,
883			minimum_frequency, s->frequency_scale, s->deviation);
884		✗	s->nb_consumed_samples = FFMIN(s->nb_consumed_samples, 65536);
885
886		✗	s->nb_threads = FFMIN(s->frequency_band_count, ff_filter_get_nb_threads(ctx));
887		✗	s->nb_channels = inlink->ch_layout.nb_channels;
888		✗	s->old_pts = AV_NOPTS_VALUE;
889		✗	s->eof_pts = AV_NOPTS_VALUE;
890
891		✗	s->input_sample_count = 1 << (32 - ff_clz(s->nb_consumed_samples));
892		✗	s->input_padding_size = 1 << (32 - ff_clz(s->input_sample_count));
893		✗	s->output_sample_count = FFMAX(1, av_rescale(s->input_sample_count, s->pps, inlink->sample_rate));
894		✗	s->output_padding_size = 1 << (32 - ff_clz(s->output_sample_count));
895
896		✗	s->hop_size = s->input_sample_count;
897		✗	s->ihop_size = s->output_padding_size >> 1;
898
899		✗	outlink->w = s->w;
900		✗	outlink->h = s->h;
901		✗	outlink->sample_aspect_ratio = (AVRational){1,1};
902
903		✗	s->fft_size = FFALIGN(s->input_padding_size, av_cpu_max_align());
904		✗	s->ifft_size = FFALIGN(s->output_padding_size, av_cpu_max_align());
905
906		✗	s->fft = av_calloc(s->nb_threads, sizeof(*s->fft));
907		✗	if (!s->fft)
908		✗	return AVERROR(ENOMEM);
909
910		✗	for (int n = 0; n < s->nb_threads; n++) {
911		✗	ret = av_tx_init(&s->fft[n], &s->tx_fn, AV_TX_FLOAT_FFT, 0, s->input_padding_size, &scale, 0);
912		✗	if (ret < 0)
913		✗	return ret;
914			}
915
916		✗	s->ifft = av_calloc(s->nb_threads, sizeof(*s->ifft));
917		✗	if (!s->ifft)
918		✗	return AVERROR(ENOMEM);
919
920		✗	for (int n = 0; n < s->nb_threads; n++) {
921		✗	ret = av_tx_init(&s->ifft[n], &s->itx_fn, AV_TX_FLOAT_FFT, 1, s->output_padding_size, &scale, 0);
922		✗	if (ret < 0)
923		✗	return ret;
924			}
925
926		✗	s->outpicref = ff_get_video_buffer(outlink, outlink->w, outlink->h);
927		✗	s->fft_in = ff_get_audio_buffer(inlink, s->fft_size * 2);
928		✗	s->fft_out = ff_get_audio_buffer(inlink, s->fft_size * 2);
929		✗	s->dst_x = av_frame_alloc();
930		✗	s->src_x = av_frame_alloc();
931		✗	s->kernel = av_calloc(s->frequency_band_count, sizeof(*s->kernel));
932		✗	s->cache = ff_get_audio_buffer(inlink, s->hop_size);
933		✗	s->over = ff_get_audio_buffer(inlink, s->frequency_band_count * 2 * s->ihop_size);
934		✗	s->bh_out = ff_get_audio_buffer(inlink, s->frequency_band_count);
935		✗	s->ifft_in = av_frame_alloc();
936		✗	s->ifft_out = av_frame_alloc();
937		✗	s->ch_out = av_frame_alloc();
938		✗	s->index = av_calloc(s->input_padding_size, sizeof(*s->index));
939		✗	s->kernel_start = av_calloc(s->frequency_band_count, sizeof(*s->kernel_start));
940		✗	s->kernel_stop = av_calloc(s->frequency_band_count, sizeof(*s->kernel_stop));
941		✗	if (!s->outpicref || !s->fft_in || !s->fft_out || !s->src_x || !s->dst_x || !s->over ||
942		✗	!s->ifft_in || !s->ifft_out || !s->kernel_start || !s->kernel_stop || !s->ch_out ||
943		✗	!s->cache || !s->index || !s->bh_out || !s->kernel)
944		✗	return AVERROR(ENOMEM);
945
946		✗	s->ch_out->format = inlink->format;
947		✗	s->ch_out->nb_samples = 2 * s->ihop_size * inlink->ch_layout.nb_channels;
948		✗	s->ch_out->ch_layout.nb_channels = s->frequency_band_count;
949		✗	ret = av_frame_get_buffer(s->ch_out, 0);
950		✗	if (ret < 0)
951		✗	return ret;
952
953		✗	s->ifft_in->format = inlink->format;
954		✗	s->ifft_in->nb_samples = s->ifft_size * 2;
955		✗	s->ifft_in->ch_layout.nb_channels = s->nb_threads;
956		✗	ret = av_frame_get_buffer(s->ifft_in, 0);
957		✗	if (ret < 0)
958		✗	return ret;
959
960		✗	s->ifft_out->format = inlink->format;
961		✗	s->ifft_out->nb_samples = s->ifft_size * 2;
962		✗	s->ifft_out->ch_layout.nb_channels = s->nb_threads;
963		✗	ret = av_frame_get_buffer(s->ifft_out, 0);
964		✗	if (ret < 0)
965		✗	return ret;
966
967		✗	s->src_x->format = inlink->format;
968		✗	s->src_x->nb_samples = s->fft_size * 2;
969		✗	s->src_x->ch_layout.nb_channels = s->nb_threads;
970		✗	ret = av_frame_get_buffer(s->src_x, 0);
971		✗	if (ret < 0)
972		✗	return ret;
973
974		✗	s->dst_x->format = inlink->format;
975		✗	s->dst_x->nb_samples = s->fft_size * 2;
976		✗	s->dst_x->ch_layout.nb_channels = s->nb_threads;
977		✗	ret = av_frame_get_buffer(s->dst_x, 0);
978		✗	if (ret < 0)
979		✗	return ret;
980
981		✗	s->outpicref->sample_aspect_ratio = (AVRational){1,1};
982
983		✗	for (int y = 0; y < outlink->h; y++) {
984		✗	memset(s->outpicref->data[0] + y * s->outpicref->linesize[0], 0, outlink->w);
985		✗	memset(s->outpicref->data[1] + y * s->outpicref->linesize[1], 128, outlink->w);
986		✗	memset(s->outpicref->data[2] + y * s->outpicref->linesize[2], 128, outlink->w);
987		✗	if (s->outpicref->data[3])
988		✗	memset(s->outpicref->data[3] + y * s->outpicref->linesize[3], 0, outlink->w);
989			}
990
991		✗	s->outpicref->color_range = AVCOL_RANGE_JPEG;
992
993		✗	factor = s->input_padding_size / (float)inlink->sample_rate;
994		✗	for (int n = 0; n < s->frequency_band_count; n++) {
995		✗	s->frequency_band[2*n ] *= factor;
996		✗	s->frequency_band[2*n+1] *= factor;
997			}
998
999		✗	av_log(ctx, AV_LOG_DEBUG, "factor: %f\n", factor);
1000		✗	av_log(ctx, AV_LOG_DEBUG, "nb_consumed_samples: %d\n", s->nb_consumed_samples);
1001		✗	av_log(ctx, AV_LOG_DEBUG, "hop_size: %d\n", s->hop_size);
1002		✗	av_log(ctx, AV_LOG_DEBUG, "ihop_size: %d\n", s->ihop_size);
1003		✗	av_log(ctx, AV_LOG_DEBUG, "input_sample_count: %d\n", s->input_sample_count);
1004		✗	av_log(ctx, AV_LOG_DEBUG, "input_padding_size: %d\n", s->input_padding_size);
1005		✗	av_log(ctx, AV_LOG_DEBUG, "output_sample_count: %d\n", s->output_sample_count);
1006		✗	av_log(ctx, AV_LOG_DEBUG, "output_padding_size: %d\n", s->output_padding_size);
1007
1008		✗	switch (s->direction) {
1009		✗	case DIRECTION_LR:
1010			case DIRECTION_UD:
1011		✗	s->pos = s->bar_size;
1012		✗	break;
1013		✗	case DIRECTION_RL:
1014			case DIRECTION_DU:
1015		✗	s->pos = s->sono_size;
1016		✗	break;
1017			}
1018
1019		✗	s->auto_frame_rate = av_make_q(inlink->sample_rate, s->hop_size);
1020		✗	if (strcmp(s->rate_str, "auto")) {
1021		✗	ret = av_parse_video_rate(&s->frame_rate, s->rate_str);
1022		✗	if (ret < 0)
1023		✗	return ret;
1024			} else {
1025		✗	s->frame_rate = s->auto_frame_rate;
1026			}
1027		✗	l->frame_rate = s->frame_rate;
1028		✗	outlink->time_base = av_inv_q(l->frame_rate);
1029
1030		✗	ret = compute_kernel(ctx);
1031		✗	if (ret < 0)
1032		✗	return ret;
1033
1034		✗	return 0;
1035			}
1036
1037		✗	static int output_frame(AVFilterContext *ctx)
1038			{
1039		✗	AVFilterLink *outlink = ctx->outputs[0];
1040		✗	AVFilterLink *inlink = ctx->inputs[0];
1041		✗	ShowCWTContext *s = ctx->priv;
1042		✗	const int nb_planes = 3 + (s->outpicref->data[3] != NULL);
1043			int ret;
1044
1045		✗	switch (s->slide) {
1046		✗	case SLIDE_SCROLL:
1047		✗	switch (s->direction) {
1048		✗	case DIRECTION_UD:
1049		✗	for (int p = 0; p < nb_planes; p++) {
1050		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1051
1052		✗	for (int y = s->h - 1; y > s->bar_size; y--) {
1053		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1054
1055		✗	memmove(dst, dst - linesize, s->w);
1056			}
1057			}
1058		✗	break;
1059		✗	case DIRECTION_DU:
1060		✗	for (int p = 0; p < nb_planes; p++) {
1061		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1062
1063		✗	for (int y = 0; y < s->sono_size; y++) {
1064		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1065
1066		✗	memmove(dst, dst + linesize, s->w);
1067			}
1068			}
1069		✗	break;
1070			}
1071		✗	break;
1072			}
1073
1074		✗	ff_filter_execute(ctx, draw, NULL, NULL, s->nb_threads);
1075
1076		✗	switch (s->slide) {
1077		✗	case SLIDE_REPLACE:
1078			case SLIDE_FRAME:
1079		✗	switch (s->direction) {
1080		✗	case DIRECTION_LR:
1081		✗	s->pos++;
1082		✗	if (s->pos >= s->w) {
1083		✗	s->pos = s->bar_size;
1084		✗	s->new_frame = 1;
1085			}
1086		✗	break;
1087		✗	case DIRECTION_RL:
1088		✗	s->pos--;
1089		✗	if (s->pos < 0) {
1090		✗	s->pos = s->sono_size;
1091		✗	s->new_frame = 1;
1092			}
1093		✗	break;
1094		✗	case DIRECTION_UD:
1095		✗	s->pos++;
1096		✗	if (s->pos >= s->h) {
1097		✗	s->pos = s->bar_size;
1098		✗	s->new_frame = 1;
1099			}
1100		✗	break;
1101		✗	case DIRECTION_DU:
1102		✗	s->pos--;
1103		✗	if (s->pos < 0) {
1104		✗	s->pos = s->sono_size;
1105		✗	s->new_frame = 1;
1106			}
1107		✗	break;
1108			}
1109		✗	break;
1110		✗	case SLIDE_SCROLL:
1111		✗	switch (s->direction) {
1112		✗	case DIRECTION_UD:
1113			case DIRECTION_LR:
1114		✗	s->pos = s->bar_size;
1115		✗	break;
1116		✗	case DIRECTION_RL:
1117			case DIRECTION_DU:
1118		✗	s->pos = s->sono_size;
1119		✗	break;
1120			}
1121		✗	break;
1122			}
1123
1124		✗	if (s->slide == SLIDE_FRAME && s->eof) {
1125		✗	switch (s->direction) {
1126		✗	case DIRECTION_LR:
1127		✗	for (int p = 0; p < nb_planes; p++) {
1128		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1129		✗	const int size = s->w - s->pos;
1130		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1131		✗	const int x = s->pos;
1132
1133		✗	for (int y = 0; y < s->h; y++) {
1134		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize + x;
1135
1136		✗	memset(dst, fill, size);
1137			}
1138			}
1139		✗	break;
1140		✗	case DIRECTION_RL:
1141		✗	for (int p = 0; p < nb_planes; p++) {
1142		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1143		✗	const int size = s->w - s->pos;
1144		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1145
1146		✗	for (int y = 0; y < s->h; y++) {
1147		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1148
1149		✗	memset(dst, fill, size);
1150			}
1151			}
1152		✗	break;
1153		✗	case DIRECTION_UD:
1154		✗	for (int p = 0; p < nb_planes; p++) {
1155		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1156		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1157
1158		✗	for (int y = s->pos; y < s->h; y++) {
1159		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1160
1161		✗	memset(dst, fill, s->w);
1162			}
1163			}
1164		✗	break;
1165		✗	case DIRECTION_DU:
1166		✗	for (int p = 0; p < nb_planes; p++) {
1167		✗	ptrdiff_t linesize = s->outpicref->linesize[p];
1168		✗	const int fill = p > 0 && p < 3 ? 128 : 0;
1169
1170		✗	for (int y = s->h - s->pos; y >= 0; y--) {
1171		✗	uint8_t *dst = s->outpicref->data[p] + y * linesize;
1172
1173		✗	memset(dst, fill, s->w);
1174			}
1175			}
1176		✗	break;
1177			}
1178			}
1179
1180		✗	s->new_frame = s->slide == SLIDE_FRAME && (s->new_frame || s->eof);
1181
1182		✗	if (s->slide != SLIDE_FRAME || s->new_frame == 1) {
1183		✗	int64_t pts_offset = s->new_frame ? 0LL : av_rescale(s->ihop_index, s->hop_size, s->ihop_size);
1184		✗	const int offset = (s->input_padding_size - s->hop_size) >> 1;
1185
1186		✗	pts_offset = av_rescale_q(pts_offset - offset, av_make_q(1, inlink->sample_rate), inlink->time_base);
1187		✗	s->outpicref->pts = av_rescale_q(s->in_pts + pts_offset, inlink->time_base, outlink->time_base);
1188		✗	s->outpicref->duration = 1;
1189			}
1190
1191		✗	s->ihop_index++;
1192		✗	if (s->ihop_index >= s->ihop_size)
1193		✗	s->ihop_index = s->hop_index = 0;
1194
1195		✗	if (s->slide == SLIDE_FRAME && s->new_frame == 0)
1196		✗	return 1;
1197
1198		✗	if (s->old_pts < s->outpicref->pts) {
1199		✗	AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
1200		✗	if (!out)
1201		✗	return AVERROR(ENOMEM);
1202		✗	ret = av_frame_copy_props(out, s->outpicref);
1203		✗	if (ret < 0)
1204		✗	goto fail;
1205		✗	ret = av_frame_copy(out, s->outpicref);
1206		✗	if (ret < 0)
1207		✗	goto fail;
1208		✗	s->old_pts = s->outpicref->pts;
1209		✗	s->new_frame = 0;
1210		✗	ret = ff_filter_frame(outlink, out);
1211		✗	if (ret <= 0)
1212		✗	return ret;
1213		✗	fail:
1214		✗	av_frame_free(&out);
1215		✗	return ret;
1216			}
1217
1218		✗	return 1;
1219			}
1220
1221		✗	static int run_channels_cwt_prepare(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
1222			{
1223		✗	ShowCWTContext *s = ctx->priv;
1224		✗	const int count = s->nb_channels;
1225		✗	const int start = (count * jobnr) / nb_jobs;
1226		✗	const int end = (count * (jobnr+1)) / nb_jobs;
1227
1228		✗	for (int ch = start; ch < end; ch++)
1229		✗	run_channel_cwt_prepare(ctx, arg, jobnr, ch);
1230
1231		✗	return 0;
1232			}
1233
1234		✗	static int activate(AVFilterContext *ctx)
1235			{
1236		✗	AVFilterLink *inlink = ctx->inputs[0];
1237		✗	AVFilterLink *outlink = ctx->outputs[0];
1238		✗	ShowCWTContext *s = ctx->priv;
1239		✗	int ret = 0, status;
1240			int64_t pts;
1241
1242		✗	FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
1243
1244		✗	if (s->outpicref) {
1245		✗	AVFrame *fin = NULL;
1246
1247		✗	if (s->hop_index < s->hop_size) {
1248		✗	if (!s->eof) {
1249		✗	ret = ff_inlink_consume_samples(inlink, 1, s->hop_size - s->hop_index, &fin);
1250		✗	if (ret < 0)
1251		✗	return ret;
1252			}
1253
1254		✗	if (ret > 0 || s->eof) {
1255		✗	ff_filter_execute(ctx, run_channels_cwt_prepare, fin, NULL,
1256		✗	FFMIN(s->nb_threads, s->nb_channels));
1257		✗	if (fin) {
1258		✗	if (s->hop_index == 0) {
1259		✗	s->in_pts = fin->pts;
1260		✗	if (s->old_pts == AV_NOPTS_VALUE)
1261		✗	s->old_pts = av_rescale_q(s->in_pts, inlink->time_base, outlink->time_base) - 1;
1262			}
1263		✗	s->hop_index += fin->nb_samples;
1264		✗	av_frame_free(&fin);
1265			} else {
1266		✗	s->hop_index = s->hop_size;
1267			}
1268			}
1269			}
1270
1271		✗	if (s->hop_index >= s->hop_size || s->ihop_index > 0) {
1272		✗	for (int ch = 0; ch < s->nb_channels && s->ihop_index == 0; ch++) {
1273		✗	ff_filter_execute(ctx, run_channel_cwt, (void *)&ch, NULL,
1274			s->nb_threads);
1275			}
1276
1277		✗	ret = output_frame(ctx);
1278		✗	if (ret != 1)
1279		✗	return ret;
1280			}
1281			}
1282
1283		✗	if (s->eof) {
1284		✗	if (s->slide == SLIDE_FRAME)
1285		✗	ret = output_frame(ctx);
1286		✗	ff_outlink_set_status(outlink, AVERROR_EOF, s->eof_pts);
1287		✗	return ret;
1288			}
1289
1290		✗	if (!s->eof && ff_inlink_acknowledge_status(inlink, &status, &pts)) {
1291		✗	if (status == AVERROR_EOF) {
1292		✗	s->eof = 1;
1293		✗	ff_filter_set_ready(ctx, 10);
1294		✗	s->eof_pts = av_rescale_q(pts, inlink->time_base, outlink->time_base);
1295		✗	return 0;
1296			}
1297			}
1298
1299		✗	if (ff_inlink_queued_samples(inlink) > 0 || s->ihop_index ||
1300		✗	s->hop_index >= s->hop_size || s->eof) {
1301		✗	ff_filter_set_ready(ctx, 10);
1302		✗	return 0;
1303			}
1304
1305		✗	if (ff_outlink_frame_wanted(outlink)) {
1306		✗	ff_inlink_request_frame(inlink);
1307		✗	return 0;
1308			}
1309
1310		✗	return FFERROR_NOT_READY;
1311			}
1312
1313			static const AVFilterPad showcwt_outputs[] = {
1314			{
1315			.name = "default",
1316			.type = AVMEDIA_TYPE_VIDEO,
1317			.config_props = config_output,
1318			},
1319			};
1320
1321			const AVFilter ff_avf_showcwt = {
1322			.name = "showcwt",
1323			.description = NULL_IF_CONFIG_SMALL("Convert input audio to a CWT (Continuous Wavelet Transform) spectrum video output."),
1324			.uninit = uninit,
1325			.priv_size = sizeof(ShowCWTContext),
1326			FILTER_INPUTS(ff_audio_default_filterpad),
1327			FILTER_OUTPUTS(showcwt_outputs),
1328			FILTER_QUERY_FUNC2(query_formats),
1329			.activate = activate,
1330			.priv_class = &showcwt_class,
1331			.flags = AVFILTER_FLAG_SLICE_THREADS,
1332			};
1333