FFmpeg coverage


Directory: ../../../ffmpeg/
File: src/libavfilter/avf_showcwt.c
Date: 2024-07-17 14:05:47
Exec Total Coverage
Lines: 0 813 0.0%
Functions: 0 13 0.0%
Branches: 0 419 0.0%

Line Branch Exec Source
1 /*
2 * Copyright (c) 2022 Paul B Mahol
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include <float.h>
22 #include <math.h>
23
24 #include "libavutil/mem.h"
25 #include "libavutil/tx.h"
26 #include "libavutil/channel_layout.h"
27 #include "libavutil/float_dsp.h"
28 #include "libavutil/cpu.h"
29 #include "libavutil/opt.h"
30 #include "libavutil/parseutils.h"
31 #include "audio.h"
32 #include "formats.h"
33 #include "video.h"
34 #include "avfilter.h"
35 #include "filters.h"
36 #include "internal.h"
37
38 enum FrequencyScale {
39 FSCALE_LINEAR,
40 FSCALE_LOG,
41 FSCALE_BARK,
42 FSCALE_MEL,
43 FSCALE_ERBS,
44 FSCALE_SQRT,
45 FSCALE_CBRT,
46 FSCALE_QDRT,
47 FSCALE_FM,
48 NB_FSCALE
49 };
50
51 enum IntensityScale {
52 ISCALE_LOG,
53 ISCALE_LINEAR,
54 ISCALE_SQRT,
55 ISCALE_CBRT,
56 ISCALE_QDRT,
57 NB_ISCALE
58 };
59
60 enum DirectionMode {
61 DIRECTION_LR,
62 DIRECTION_RL,
63 DIRECTION_UD,
64 DIRECTION_DU,
65 NB_DIRECTION
66 };
67
68 enum SlideMode {
69 SLIDE_REPLACE,
70 SLIDE_SCROLL,
71 SLIDE_FRAME,
72 NB_SLIDE
73 };
74
75 typedef struct ShowCWTContext {
76 const AVClass *class;
77 int w, h;
78 int mode;
79 char *rate_str;
80 AVRational auto_frame_rate;
81 AVRational frame_rate;
82 AVTXContext **fft, **ifft;
83 av_tx_fn tx_fn, itx_fn;
84 int fft_size, ifft_size;
85 int pos;
86 int64_t in_pts;
87 int64_t old_pts;
88 int64_t eof_pts;
89 float *frequency_band;
90 AVComplexFloat **kernel;
91 unsigned *index;
92 int *kernel_start, *kernel_stop;
93 AVFrame *cache;
94 AVFrame *outpicref;
95 AVFrame *fft_in;
96 AVFrame *fft_out;
97 AVFrame *dst_x;
98 AVFrame *src_x;
99 AVFrame *ifft_in;
100 AVFrame *ifft_out;
101 AVFrame *ch_out;
102 AVFrame *over;
103 AVFrame *bh_out;
104 int nb_threads;
105 int nb_channels;
106 int nb_consumed_samples;
107 int pps;
108 int eof;
109 int slide;
110 int new_frame;
111 int direction;
112 int hop_size, ihop_size;
113 int hop_index, ihop_index;
114 int input_padding_size, output_padding_size;
115 int input_sample_count, output_sample_count;
116 int frequency_band_count;
117 float logarithmic_basis;
118 int intensity_scale;
119 int frequency_scale;
120 float minimum_frequency, maximum_frequency;
121 float minimum_intensity, maximum_intensity;
122 float deviation;
123 float bar_ratio;
124 int bar_size;
125 int sono_size;
126 float rotation;
127
128 AVFloatDSPContext *fdsp;
129 } ShowCWTContext;
130
131 #define OFFSET(x) offsetof(ShowCWTContext, x)
132 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
133
134 static const AVOption showcwt_options[] = {
135 { "size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "640x512"}, 0, 0, FLAGS },
136 { "s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "640x512"}, 0, 0, FLAGS },
137 { "rate", "set video rate", OFFSET(rate_str), AV_OPT_TYPE_STRING, {.str = "25"}, 0, 0, FLAGS },
138 { "r", "set video rate", OFFSET(rate_str), AV_OPT_TYPE_STRING, {.str = "25"}, 0, 0, FLAGS },
139 { "scale", "set frequency scale", OFFSET(frequency_scale), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_FSCALE-1, FLAGS, .unit = "scale" },
140 { "linear", "linear", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_LINEAR}, 0, 0, FLAGS, .unit = "scale" },
141 { "log", "logarithmic", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_LOG}, 0, 0, FLAGS, .unit = "scale" },
142 { "bark", "bark", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_BARK}, 0, 0, FLAGS, .unit = "scale" },
143 { "mel", "mel", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_MEL}, 0, 0, FLAGS, .unit = "scale" },
144 { "erbs", "erbs", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_ERBS}, 0, 0, FLAGS, .unit = "scale" },
145 { "sqrt", "sqrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_SQRT}, 0, 0, FLAGS, .unit = "scale" },
146 { "cbrt", "cbrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_CBRT}, 0, 0, FLAGS, .unit = "scale" },
147 { "qdrt", "qdrt", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_QDRT}, 0, 0, FLAGS, .unit = "scale" },
148 { "fm", "fm", 0, AV_OPT_TYPE_CONST,{.i64=FSCALE_FM}, 0, 0, FLAGS, .unit = "scale" },
149 { "iscale", "set intensity scale", OFFSET(intensity_scale),AV_OPT_TYPE_INT, {.i64=0}, 0, NB_ISCALE-1, FLAGS, .unit = "iscale" },
150 { "linear", "linear", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_LINEAR}, 0, 0, FLAGS, .unit = "iscale" },
151 { "log", "logarithmic", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_LOG}, 0, 0, FLAGS, .unit = "iscale" },
152 { "sqrt", "sqrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_SQRT}, 0, 0, FLAGS, .unit = "iscale" },
153 { "cbrt", "cbrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_CBRT}, 0, 0, FLAGS, .unit = "iscale" },
154 { "qdrt", "qdrt", 0, AV_OPT_TYPE_CONST,{.i64=ISCALE_QDRT}, 0, 0, FLAGS, .unit = "iscale" },
155 { "min", "set minimum frequency", OFFSET(minimum_frequency), AV_OPT_TYPE_FLOAT, {.dbl = 20.}, 1, 192000, FLAGS },
156 { "max", "set maximum frequency", OFFSET(maximum_frequency), AV_OPT_TYPE_FLOAT, {.dbl = 20000.}, 1, 192000, FLAGS },
157 { "imin", "set minimum intensity", OFFSET(minimum_intensity), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
158 { "imax", "set maximum intensity", OFFSET(maximum_intensity), AV_OPT_TYPE_FLOAT, {.dbl = 1.}, 0, 1, FLAGS },
159 { "logb", "set logarithmic basis", OFFSET(logarithmic_basis), AV_OPT_TYPE_FLOAT, {.dbl = 0.0001}, 0, 1, FLAGS },
160 { "deviation", "set frequency deviation", OFFSET(deviation), AV_OPT_TYPE_FLOAT, {.dbl = 1.}, 0, 100, FLAGS },
161 { "pps", "set pixels per second", OFFSET(pps), AV_OPT_TYPE_INT, {.i64 = 64}, 1, 1024, FLAGS },
162 { "mode", "set output mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=0}, 0, 4, FLAGS, .unit = "mode" },
163 { "magnitude", "magnitude", 0, AV_OPT_TYPE_CONST,{.i64=0}, 0, 0, FLAGS, .unit = "mode" },
164 { "phase", "phase", 0, AV_OPT_TYPE_CONST,{.i64=1}, 0, 0, FLAGS, .unit = "mode" },
165 { "magphase", "magnitude+phase", 0, AV_OPT_TYPE_CONST,{.i64=2}, 0, 0, FLAGS, .unit = "mode" },
166 { "channel", "color per channel", 0, AV_OPT_TYPE_CONST,{.i64=3}, 0, 0, FLAGS, .unit = "mode" },
167 { "stereo", "stereo difference", 0, AV_OPT_TYPE_CONST,{.i64=4}, 0, 0, FLAGS, .unit = "mode" },
168 { "slide", "set slide mode", OFFSET(slide), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_SLIDE-1, FLAGS, .unit = "slide" },
169 { "replace", "replace", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_REPLACE},0, 0, FLAGS, .unit = "slide" },
170 { "scroll", "scroll", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_SCROLL}, 0, 0, FLAGS, .unit = "slide" },
171 { "frame", "frame", 0, AV_OPT_TYPE_CONST,{.i64=SLIDE_FRAME}, 0, 0, FLAGS, .unit = "slide" },
172 { "direction", "set direction mode", OFFSET(direction), AV_OPT_TYPE_INT, {.i64=0}, 0, NB_DIRECTION-1, FLAGS, .unit = "direction" },
173 { "lr", "left to right", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_LR}, 0, 0, FLAGS, .unit = "direction" },
174 { "rl", "right to left", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_RL}, 0, 0, FLAGS, .unit = "direction" },
175 { "ud", "up to down", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_UD}, 0, 0, FLAGS, .unit = "direction" },
176 { "du", "down to up", 0, AV_OPT_TYPE_CONST,{.i64=DIRECTION_DU}, 0, 0, FLAGS, .unit = "direction" },
177 { "bar", "set bargraph ratio", OFFSET(bar_ratio), AV_OPT_TYPE_FLOAT, {.dbl = 0.}, 0, 1, FLAGS },
178 { "rotation", "set color rotation", OFFSET(rotation), AV_OPT_TYPE_FLOAT, {.dbl = 0}, -1, 1, FLAGS },
179 { NULL }
180 };
181
182 AVFILTER_DEFINE_CLASS(showcwt);
183
184 static av_cold void uninit(AVFilterContext *ctx)
185 {
186 ShowCWTContext *s = ctx->priv;
187
188 av_freep(&s->frequency_band);
189 av_freep(&s->kernel_start);
190 av_freep(&s->kernel_stop);
191 av_freep(&s->index);
192
193 av_frame_free(&s->cache);
194 av_frame_free(&s->outpicref);
195 av_frame_free(&s->fft_in);
196 av_frame_free(&s->fft_out);
197 av_frame_free(&s->dst_x);
198 av_frame_free(&s->src_x);
199 av_frame_free(&s->ifft_in);
200 av_frame_free(&s->ifft_out);
201 av_frame_free(&s->ch_out);
202 av_frame_free(&s->over);
203 av_frame_free(&s->bh_out);
204
205 if (s->fft) {
206 for (int n = 0; n < s->nb_threads; n++)
207 av_tx_uninit(&s->fft[n]);
208 av_freep(&s->fft);
209 }
210
211 if (s->ifft) {
212 for (int n = 0; n < s->nb_threads; n++)
213 av_tx_uninit(&s->ifft[n]);
214 av_freep(&s->ifft);
215 }
216
217 if (s->kernel) {
218 for (int n = 0; n < s->frequency_band_count; n++)
219 av_freep(&s->kernel[n]);
220 }
221 av_freep(&s->kernel);
222
223 av_freep(&s->fdsp);
224 }
225
226 static int query_formats(AVFilterContext *ctx)
227 {
228 AVFilterFormats *formats = NULL;
229 AVFilterChannelLayouts *layouts = NULL;
230 AVFilterLink *inlink = ctx->inputs[0];
231 AVFilterLink *outlink = ctx->outputs[0];
232 static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE };
233 static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_NONE };
234 int ret;
235
236 formats = ff_make_format_list(sample_fmts);
237 if ((ret = ff_formats_ref(formats, &inlink->outcfg.formats)) < 0)
238 return ret;
239
240 layouts = ff_all_channel_counts();
241 if ((ret = ff_channel_layouts_ref(layouts, &inlink->outcfg.channel_layouts)) < 0)
242 return ret;
243
244 formats = ff_all_samplerates();
245 if ((ret = ff_formats_ref(formats, &inlink->outcfg.samplerates)) < 0)
246 return ret;
247
248 formats = ff_make_format_list(pix_fmts);
249 if ((ret = ff_formats_ref(formats, &outlink->incfg.formats)) < 0)
250 return ret;
251
252 return 0;
253 }
254
255 static float frequency_band(float *frequency_band,
256 int frequency_band_count,
257 float frequency_range,
258 float frequency_offset,
259 int frequency_scale, float deviation)
260 {
261 float ret = 0.f;
262
263 deviation = sqrtf(deviation / (4.f * M_PI)); // Heisenberg Gabor Limit
264 for (int y = 0; y < frequency_band_count; y++) {
265 float frequency = frequency_range * (1.f - (float)y / frequency_band_count) + frequency_offset;
266 float frequency_derivative = frequency_range / frequency_band_count;
267
268 switch (frequency_scale) {
269 case FSCALE_LOG:
270 frequency = powf(2.f, frequency);
271 frequency_derivative *= logf(2.f) * frequency;
272 break;
273 case FSCALE_BARK:
274 frequency = 600.f * sinhf(frequency / 6.f);
275 frequency_derivative *= sqrtf(frequency * frequency + 360000.f) / 6.f;
276 break;
277 case FSCALE_MEL:
278 frequency = 700.f * (powf(10.f, frequency / 2595.f) - 1.f);
279 frequency_derivative *= (frequency + 700.f) * logf(10.f) / 2595.f;
280 break;
281 case FSCALE_ERBS:
282 frequency = 676170.4f / (47.06538f - expf(frequency * 0.08950404f)) - 14678.49f;
283 frequency_derivative *= (frequency * frequency + 14990.4f * frequency + 4577850.f) / 160514.f;
284 break;
285 case FSCALE_SQRT:
286 frequency = frequency * frequency;
287 frequency_derivative *= 2.f * sqrtf(frequency);
288 break;
289 case FSCALE_CBRT:
290 frequency = frequency * frequency * frequency;
291 frequency_derivative *= 3.f * powf(frequency, 2.f / 3.f);
292 break;
293 case FSCALE_QDRT:
294 frequency = frequency * frequency * frequency * frequency;
295 frequency_derivative *= 4.f * powf(frequency, 3.f / 4.f);
296 break;
297 case FSCALE_FM:
298 frequency = 2.f * powf(frequency, 3.f / 2.f) / 3.f;
299 frequency_derivative *= sqrtf(frequency);
300 break;
301 }
302
303 frequency_band[y*2 ] = frequency;
304 frequency_band[y*2+1] = frequency_derivative * deviation;
305
306 ret = 1.f / (frequency_derivative * deviation);
307 }
308
309 return ret;
310 }
311
312 static float remap_log(ShowCWTContext *s, float value, int iscale, float log_factor)
313 {
314 const float max = s->maximum_intensity;
315 const float min = s->minimum_intensity;
316 float ret;
317
318 value += min;
319
320 switch (iscale) {
321 case ISCALE_LINEAR:
322 ret = max - expf(value / log_factor);
323 break;
324 case ISCALE_LOG:
325 value = logf(value) * log_factor;
326 ret = max - av_clipf(value, 0.f, 1.f);
327 break;
328 case ISCALE_SQRT:
329 value = max - expf(value / log_factor);
330 ret = sqrtf(value);
331 break;
332 case ISCALE_CBRT:
333 value = max - expf(value / log_factor);
334 ret = cbrtf(value);
335 break;
336 case ISCALE_QDRT:
337 value = max - expf(value / log_factor);
338 ret = powf(value, 0.25f);
339 break;
340 }
341
342 return av_clipf(ret, 0.f, 1.f);
343 }
344
345 static int run_channel_cwt_prepare(AVFilterContext *ctx, void *arg, int jobnr, int ch)
346 {
347 ShowCWTContext *s = ctx->priv;
348 const int hop_size = s->hop_size;
349 AVFrame *fin = arg;
350 float *cache = (float *)s->cache->extended_data[ch];
351 AVComplexFloat *src = (AVComplexFloat *)s->fft_in->extended_data[ch];
352 AVComplexFloat *dst = (AVComplexFloat *)s->fft_out->extended_data[ch];
353 const int offset = (s->input_padding_size - hop_size) >> 1;
354
355 if (fin) {
356 const float *input = (const float *)fin->extended_data[ch];
357 const int offset = s->hop_size - fin->nb_samples;
358
359 memmove(cache, &cache[fin->nb_samples], offset * sizeof(float));
360 memcpy(&cache[offset], input, fin->nb_samples * sizeof(float));
361 }
362
363 if (fin && s->hop_index + fin->nb_samples < hop_size)
364 return 0;
365
366 memset(src, 0, sizeof(float) * s->fft_size);
367 for (int n = 0; n < hop_size; n++)
368 src[n+offset].re = cache[n];
369
370 s->tx_fn(s->fft[jobnr], dst, src, sizeof(*src));
371
372 return 0;
373 }
374
375 #define DRAW_BAR_COLOR(x) \
376 do { \
377 if (Y <= ht) { \
378 dstY[x] = 0; \
379 dstU[x] = 128; \
380 dstV[x] = 128; \
381 } else { \
382 float mul = (Y - ht) * bh[0]; \
383 dstY[x] = av_clip_uint8(lrintf(Y * mul * 255.f)); \
384 dstU[x] = av_clip_uint8(lrintf((U-0.5f) * 128.f + 128)); \
385 dstV[x] = av_clip_uint8(lrintf((V-0.5f) * 128.f + 128)); \
386 } \
387 } while (0)
388
389 static void draw_bar(ShowCWTContext *s, int y,
390 float Y, float U, float V)
391 {
392 float *bh = ((float *)s->bh_out->extended_data[0]) + y;
393 const ptrdiff_t ylinesize = s->outpicref->linesize[0];
394 const ptrdiff_t ulinesize = s->outpicref->linesize[1];
395 const ptrdiff_t vlinesize = s->outpicref->linesize[2];
396 const int direction = s->direction;
397 const int sono_size = s->sono_size;
398 const int bar_size = s->bar_size;
399 const float rcp_bar_h = 1.f / bar_size;
400 uint8_t *dstY, *dstU, *dstV;
401 const int w = s->w;
402
403 bh[0] = 1.f / (Y + 0.0001f);
404 switch (direction) {
405 case DIRECTION_LR:
406 dstY = s->outpicref->data[0] + y * ylinesize;
407 dstU = s->outpicref->data[1] + y * ulinesize;
408 dstV = s->outpicref->data[2] + y * vlinesize;
409 for (int x = 0; x < bar_size; x++) {
410 float ht = (bar_size - x) * rcp_bar_h;
411 DRAW_BAR_COLOR(x);
412 }
413 break;
414 case DIRECTION_RL:
415 dstY = s->outpicref->data[0] + y * ylinesize;
416 dstU = s->outpicref->data[1] + y * ulinesize;
417 dstV = s->outpicref->data[2] + y * vlinesize;
418 for (int x = 0; x < bar_size; x++) {
419 float ht = x * rcp_bar_h;
420 DRAW_BAR_COLOR(w - bar_size + x);
421 }
422 break;
423 case DIRECTION_UD:
424 dstY = s->outpicref->data[0] + w - 1 - y;
425 dstU = s->outpicref->data[1] + w - 1 - y;
426 dstV = s->outpicref->data[2] + w - 1 - y;
427 for (int x = 0; x < bar_size; x++) {
428 float ht = (bar_size - x) * rcp_bar_h;
429 DRAW_BAR_COLOR(0);
430 dstY += ylinesize;
431 dstU += ulinesize;
432 dstV += vlinesize;
433 }
434 break;
435 case DIRECTION_DU:
436 dstY = s->outpicref->data[0] + w - 1 - y + ylinesize * sono_size;
437 dstU = s->outpicref->data[1] + w - 1 - y + ulinesize * sono_size;
438 dstV = s->outpicref->data[2] + w - 1 - y + vlinesize * sono_size;
439 for (int x = 0; x < bar_size; x++) {
440 float ht = x * rcp_bar_h;
441 DRAW_BAR_COLOR(0);
442 dstY += ylinesize;
443 dstU += ulinesize;
444 dstV += vlinesize;
445 }
446 break;
447 }
448 }
449
450 static int draw(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
451 {
452 ShowCWTContext *s = ctx->priv;
453 const ptrdiff_t ylinesize = s->outpicref->linesize[0];
454 const ptrdiff_t ulinesize = s->outpicref->linesize[1];
455 const ptrdiff_t vlinesize = s->outpicref->linesize[2];
456 const ptrdiff_t alinesize = s->outpicref->linesize[3];
457 const float log_factor = 1.f/logf(s->logarithmic_basis);
458 const int count = s->frequency_band_count;
459 const int start = (count * jobnr) / nb_jobs;
460 const int end = (count * (jobnr+1)) / nb_jobs;
461 const int nb_channels = s->nb_channels;
462 const int iscale = s->intensity_scale;
463 const int ihop_index = s->ihop_index;
464 const int ihop_size = s->ihop_size;
465 const float rotation = s->rotation;
466 const int direction = s->direction;
467 uint8_t *dstY, *dstU, *dstV, *dstA;
468 const int sono_size = s->sono_size;
469 const int bar_size = s->bar_size;
470 const int mode = s->mode;
471 const int w_1 = s->w - 1;
472 const int x = s->pos;
473 float Y, U, V;
474
475 for (int y = start; y < end; y++) {
476 const AVComplexFloat *src = ((const AVComplexFloat *)s->ch_out->extended_data[y]) +
477 0 * ihop_size + ihop_index;
478
479 if (sono_size <= 0)
480 goto skip;
481
482 switch (direction) {
483 case DIRECTION_LR:
484 case DIRECTION_RL:
485 dstY = s->outpicref->data[0] + y * ylinesize;
486 dstU = s->outpicref->data[1] + y * ulinesize;
487 dstV = s->outpicref->data[2] + y * vlinesize;
488 dstA = s->outpicref->data[3] ? s->outpicref->data[3] + y * alinesize : NULL;
489 break;
490 case DIRECTION_UD:
491 case DIRECTION_DU:
492 dstY = s->outpicref->data[0] + x * ylinesize + w_1 - y;
493 dstU = s->outpicref->data[1] + x * ulinesize + w_1 - y;
494 dstV = s->outpicref->data[2] + x * vlinesize + w_1 - y;
495 dstA = s->outpicref->data[3] ? s->outpicref->data[3] + x * alinesize + w_1 - y : NULL;
496 break;
497 }
498
499 switch (s->slide) {
500 case SLIDE_REPLACE:
501 case SLIDE_FRAME:
502 /* nothing to do here */
503 break;
504 case SLIDE_SCROLL:
505 switch (s->direction) {
506 case DIRECTION_RL:
507 memmove(dstY, dstY + 1, w_1);
508 memmove(dstU, dstU + 1, w_1);
509 memmove(dstV, dstV + 1, w_1);
510 if (dstA != NULL)
511 memmove(dstA, dstA + 1, w_1);
512 break;
513 case DIRECTION_LR:
514 memmove(dstY + 1, dstY, w_1);
515 memmove(dstU + 1, dstU, w_1);
516 memmove(dstV + 1, dstV, w_1);
517 if (dstA != NULL)
518 memmove(dstA + 1, dstA, w_1);
519 break;
520 }
521 break;
522 }
523
524 if (direction == DIRECTION_RL ||
525 direction == DIRECTION_LR) {
526 dstY += x;
527 dstU += x;
528 dstV += x;
529 if (dstA != NULL)
530 dstA += x;
531 }
532 skip:
533
534 switch (mode) {
535 case 4:
536 {
537 const AVComplexFloat *src2 = (nb_channels > 1) ? src + ihop_size: src;
538 float z, u, v;
539
540 z = hypotf(src[0].re + src2[0].re, src[0].im + src2[0].im);
541 u = hypotf(src[0].re, src[0].im);
542 v = hypotf(src2[0].re, src2[0].im);
543
544 z = remap_log(s, z, iscale, log_factor);
545 u = remap_log(s, u, iscale, log_factor);
546 v = remap_log(s, v, iscale, log_factor);
547
548 Y = z;
549 U = sinf((v - u) * M_PI_2);
550 V = sinf((u - v) * M_PI_2);
551
552 u = U * cosf(rotation * M_PI) - V * sinf(rotation * M_PI);
553 v = U * sinf(rotation * M_PI) + V * cosf(rotation * M_PI);
554
555 U = 0.5f + 0.5f * z * u;
556 V = 0.5f + 0.5f * z * v;
557
558 if (sono_size > 0) {
559 dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
560 dstU[0] = av_clip_uint8(lrintf(U * 255.f));
561 dstV[0] = av_clip_uint8(lrintf(V * 255.f));
562 if (dstA)
563 dstA[0] = dstY[0];
564 }
565
566 if (bar_size > 0)
567 draw_bar(s, y, Y, U, V);
568 }
569 break;
570 case 3:
571 {
572 const int nb_channels = s->nb_channels;
573 const float yf = 1.f / nb_channels;
574
575 Y = 0.f;
576 U = V = 0.5f;
577 for (int ch = 0; ch < nb_channels; ch++) {
578 const AVComplexFloat *srcn = src + ihop_size * ch;
579 float z;
580
581 z = hypotf(srcn[0].re, srcn[0].im);
582 z = remap_log(s, z, iscale, log_factor);
583
584 Y += z * yf;
585 U += z * yf * sinf(2.f * M_PI * (ch * yf + rotation));
586 V += z * yf * cosf(2.f * M_PI * (ch * yf + rotation));
587 }
588
589 if (sono_size > 0) {
590 dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
591 dstU[0] = av_clip_uint8(lrintf(U * 255.f));
592 dstV[0] = av_clip_uint8(lrintf(V * 255.f));
593 if (dstA)
594 dstA[0] = dstY[0];
595 }
596
597 if (bar_size > 0)
598 draw_bar(s, y, Y, U, V);
599 }
600 break;
601 case 2:
602 Y = hypotf(src[0].re, src[0].im);
603 Y = remap_log(s, Y, iscale, log_factor);
604 U = atan2f(src[0].im, src[0].re);
605 U = 0.5f + 0.5f * U * Y / M_PI;
606 V = 1.f - U;
607
608 if (sono_size > 0) {
609 dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
610 dstU[0] = av_clip_uint8(lrintf(U * 255.f));
611 dstV[0] = av_clip_uint8(lrintf(V * 255.f));
612 if (dstA)
613 dstA[0] = dstY[0];
614 }
615
616 if (bar_size > 0)
617 draw_bar(s, y, Y, U, V);
618 break;
619 case 1:
620 Y = atan2f(src[0].im, src[0].re);
621 Y = 0.5f + 0.5f * Y / M_PI;
622
623 if (sono_size > 0) {
624 dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
625 if (dstA)
626 dstA[0] = dstY[0];
627 }
628
629 if (bar_size > 0)
630 draw_bar(s, y, Y, 0.5f, 0.5f);
631 break;
632 case 0:
633 Y = hypotf(src[0].re, src[0].im);
634 Y = remap_log(s, Y, iscale, log_factor);
635
636 if (sono_size > 0) {
637 dstY[0] = av_clip_uint8(lrintf(Y * 255.f));
638 if (dstA)
639 dstA[0] = dstY[0];
640 }
641
642 if (bar_size > 0)
643 draw_bar(s, y, Y, 0.5f, 0.5f);
644 break;
645 }
646 }
647
648 return 0;
649 }
650
651 static int run_channel_cwt(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
652 {
653 ShowCWTContext *s = ctx->priv;
654 const int ch = *(int *)arg;
655 const AVComplexFloat *fft_out = (const AVComplexFloat *)s->fft_out->extended_data[ch];
656 AVComplexFloat *isrc = (AVComplexFloat *)s->ifft_in->extended_data[jobnr];
657 AVComplexFloat *idst = (AVComplexFloat *)s->ifft_out->extended_data[jobnr];
658 const int output_padding_size = s->output_padding_size;
659 const int input_padding_size = s->input_padding_size;
660 const float scale = 1.f / input_padding_size;
661 const int ihop_size = s->ihop_size;
662 const int count = s->frequency_band_count;
663 const int start = (count * jobnr) / nb_jobs;
664 const int end = (count * (jobnr+1)) / nb_jobs;
665
666 for (int y = start; y < end; y++) {
667 AVComplexFloat *chout = ((AVComplexFloat *)s->ch_out->extended_data[y]) + ch * ihop_size;
668 AVComplexFloat *over = ((AVComplexFloat *)s->over->extended_data[ch]) + y * ihop_size;
669 AVComplexFloat *dstx = (AVComplexFloat *)s->dst_x->extended_data[jobnr];
670 AVComplexFloat *srcx = (AVComplexFloat *)s->src_x->extended_data[jobnr];
671 const AVComplexFloat *kernel = s->kernel[y];
672 const unsigned *index = (const unsigned *)s->index;
673 const int kernel_start = s->kernel_start[y];
674 const int kernel_stop = s->kernel_stop[y];
675 const int kernel_range = kernel_stop - kernel_start + 1;
676 int offset;
677
678 if (kernel_start >= 0) {
679 offset = 0;
680 memcpy(srcx, fft_out + kernel_start, sizeof(*fft_out) * kernel_range);
681 } else {
682 offset = -kernel_start;
683 memcpy(srcx+offset, fft_out, sizeof(*fft_out) * (kernel_range-offset));
684 memcpy(srcx, fft_out+input_padding_size-offset, sizeof(*fft_out)*offset);
685 }
686
687 s->fdsp->vector_fmul_scalar((float *)srcx, (const float *)srcx, scale, FFALIGN(kernel_range * 2, 4));
688 s->fdsp->vector_fmul((float *)dstx, (const float *)srcx,
689 (const float *)kernel, FFALIGN(kernel_range * 2, 16));
690
691 memset(isrc, 0, sizeof(*isrc) * output_padding_size);
692 if (offset == 0) {
693 const unsigned *kindex = index + kernel_start;
694 for (int i = 0; i < kernel_range; i++) {
695 const unsigned n = kindex[i];
696
697 isrc[n].re += dstx[i].re;
698 isrc[n].im += dstx[i].im;
699 }
700 } else {
701 for (int i = 0; i < kernel_range; i++) {
702 const unsigned n = (i-kernel_start) & (output_padding_size-1);
703
704 isrc[n].re += dstx[i].re;
705 isrc[n].im += dstx[i].im;
706 }
707 }
708
709 s->itx_fn(s->ifft[jobnr], idst, isrc, sizeof(*isrc));
710
711 memcpy(chout, idst, sizeof(*chout) * ihop_size);
712 for (int n = 0; n < ihop_size; n++) {
713 chout[n].re += over[n].re;
714 chout[n].im += over[n].im;
715 }
716 memcpy(over, idst + ihop_size, sizeof(*over) * ihop_size);
717 }
718
719 return 0;
720 }
721
722 static int compute_kernel(AVFilterContext *ctx)
723 {
724 ShowCWTContext *s = ctx->priv;
725 const int size = s->input_padding_size;
726 const int output_sample_count = s->output_sample_count;
727 const int fsize = s->frequency_band_count;
728 int *kernel_start = s->kernel_start;
729 int *kernel_stop = s->kernel_stop;
730 unsigned *index = s->index;
731 int range_min = INT_MAX;
732 int range_max = 0, ret = 0;
733 float *tkernel;
734
735 tkernel = av_malloc_array(size, sizeof(*tkernel));
736 if (!tkernel)
737 return AVERROR(ENOMEM);
738
739 for (int y = 0; y < fsize; y++) {
740 AVComplexFloat *kernel = s->kernel[y];
741 int start = INT_MIN, stop = INT_MAX;
742 const float frequency = s->frequency_band[y*2];
743 const float deviation = 1.f / (s->frequency_band[y*2+1] *
744 output_sample_count);
745 const int a = FFMAX(frequency-12.f*sqrtf(1.f/deviation)-0.5f, -size);
746 const int b = FFMIN(frequency+12.f*sqrtf(1.f/deviation)-0.5f, size+a);
747 const int range = -a;
748
749 memset(tkernel, 0, size * sizeof(*tkernel));
750 for (int n = a; n < b; n++) {
751 float ff, f = n+0.5f-frequency;
752
753 ff = expf(-f*f*deviation);
754 tkernel[n+range] = ff;
755 }
756
757 for (int n = a; n < b; n++) {
758 if (tkernel[n+range] != 0.f) {
759 if (tkernel[n+range] > FLT_MIN)
760 av_log(ctx, AV_LOG_DEBUG, "out of range kernel %g\n", tkernel[n+range]);
761 start = n;
762 break;
763 }
764 }
765
766 for (int n = b; n >= a; n--) {
767 if (tkernel[n+range] != 0.f) {
768 if (tkernel[n+range] > FLT_MIN)
769 av_log(ctx, AV_LOG_DEBUG, "out of range kernel %g\n", tkernel[n+range]);
770 stop = n;
771 break;
772 }
773 }
774
775 if (start == INT_MIN || stop == INT_MAX) {
776 ret = AVERROR(EINVAL);
777 break;
778 }
779
780 kernel_start[y] = start;
781 kernel_stop[y] = stop;
782
783 kernel = av_calloc(FFALIGN(stop-start+1, 16), sizeof(*kernel));
784 if (!kernel) {
785 ret = AVERROR(ENOMEM);
786 break;
787 }
788
789 for (int n = 0; n <= stop - start; n++) {
790 kernel[n].re = tkernel[n+range+start];
791 kernel[n].im = tkernel[n+range+start];
792 }
793
794 range_min = FFMIN(range_min, stop+1-start);
795 range_max = FFMAX(range_max, stop+1-start);
796
797 s->kernel[y] = kernel;
798 }
799
800 for (int n = 0; n < size; n++)
801 index[n] = n & (s->output_padding_size - 1);
802
803 av_log(ctx, AV_LOG_DEBUG, "range_min: %d\n", range_min);
804 av_log(ctx, AV_LOG_DEBUG, "range_max: %d\n", range_max);
805
806 av_freep(&tkernel);
807
808 return ret;
809 }
810
811 static int config_output(AVFilterLink *outlink)
812 {
813 AVFilterContext *ctx = outlink->src;
814 AVFilterLink *inlink = ctx->inputs[0];
815 ShowCWTContext *s = ctx->priv;
816 const float limit_frequency = inlink->sample_rate * 0.5f;
817 float maximum_frequency = fminf(s->maximum_frequency, limit_frequency);
818 float minimum_frequency = s->minimum_frequency;
819 float scale = 1.f, factor;
820 int ret;
821
822 if (minimum_frequency >= maximum_frequency) {
823 av_log(ctx, AV_LOG_ERROR, "min frequency (%f) >= (%f) max frequency\n",
824 minimum_frequency, maximum_frequency);
825 return AVERROR(EINVAL);
826 }
827
828 uninit(ctx);
829
830 s->fdsp = avpriv_float_dsp_alloc(0);
831 if (!s->fdsp)
832 return AVERROR(ENOMEM);
833
834 switch (s->direction) {
835 case DIRECTION_LR:
836 case DIRECTION_RL:
837 s->bar_size = s->w * s->bar_ratio;
838 s->sono_size = s->w - s->bar_size;
839 s->frequency_band_count = s->h;
840 break;
841 case DIRECTION_UD:
842 case DIRECTION_DU:
843 s->bar_size = s->h * s->bar_ratio;
844 s->sono_size = s->h - s->bar_size;
845 s->frequency_band_count = s->w;
846 break;
847 }
848
849 switch (s->frequency_scale) {
850 case FSCALE_LOG:
851 minimum_frequency = logf(minimum_frequency) / logf(2.f);
852 maximum_frequency = logf(maximum_frequency) / logf(2.f);
853 break;
854 case FSCALE_BARK:
855 minimum_frequency = 6.f * asinhf(minimum_frequency / 600.f);
856 maximum_frequency = 6.f * asinhf(maximum_frequency / 600.f);
857 break;
858 case FSCALE_MEL:
859 minimum_frequency = 2595.f * log10f(1.f + minimum_frequency / 700.f);
860 maximum_frequency = 2595.f * log10f(1.f + maximum_frequency / 700.f);
861 break;
862 case FSCALE_ERBS:
863 minimum_frequency = 11.17268f * logf(1.f + (46.06538f * minimum_frequency) / (minimum_frequency + 14678.49f));
864 maximum_frequency = 11.17268f * logf(1.f + (46.06538f * maximum_frequency) / (maximum_frequency + 14678.49f));
865 break;
866 case FSCALE_SQRT:
867 minimum_frequency = sqrtf(minimum_frequency);
868 maximum_frequency = sqrtf(maximum_frequency);
869 break;
870 case FSCALE_CBRT:
871 minimum_frequency = cbrtf(minimum_frequency);
872 maximum_frequency = cbrtf(maximum_frequency);
873 break;
874 case FSCALE_QDRT:
875 minimum_frequency = powf(minimum_frequency, 0.25f);
876 maximum_frequency = powf(maximum_frequency, 0.25f);
877 break;
878 case FSCALE_FM:
879 minimum_frequency = powf(9.f * (minimum_frequency * minimum_frequency) / 4.f, 1.f / 3.f);
880 maximum_frequency = powf(9.f * (maximum_frequency * maximum_frequency) / 4.f, 1.f / 3.f);
881 break;
882 }
883
884 s->frequency_band = av_calloc(s->frequency_band_count,
885 sizeof(*s->frequency_band) * 2);
886 if (!s->frequency_band)
887 return AVERROR(ENOMEM);
888
889 s->nb_consumed_samples = inlink->sample_rate *
890 frequency_band(s->frequency_band,
891 s->frequency_band_count, maximum_frequency - minimum_frequency,
892 minimum_frequency, s->frequency_scale, s->deviation);
893 s->nb_consumed_samples = FFMIN(s->nb_consumed_samples, 65536);
894
895 s->nb_threads = FFMIN(s->frequency_band_count, ff_filter_get_nb_threads(ctx));
896 s->nb_channels = inlink->ch_layout.nb_channels;
897 s->old_pts = AV_NOPTS_VALUE;
898 s->eof_pts = AV_NOPTS_VALUE;
899
900 s->input_sample_count = 1 << (32 - ff_clz(s->nb_consumed_samples));
901 s->input_padding_size = 1 << (32 - ff_clz(s->input_sample_count));
902 s->output_sample_count = FFMAX(1, av_rescale(s->input_sample_count, s->pps, inlink->sample_rate));
903 s->output_padding_size = 1 << (32 - ff_clz(s->output_sample_count));
904
905 s->hop_size = s->input_sample_count;
906 s->ihop_size = s->output_padding_size >> 1;
907
908 outlink->w = s->w;
909 outlink->h = s->h;
910 outlink->sample_aspect_ratio = (AVRational){1,1};
911
912 s->fft_size = FFALIGN(s->input_padding_size, av_cpu_max_align());
913 s->ifft_size = FFALIGN(s->output_padding_size, av_cpu_max_align());
914
915 s->fft = av_calloc(s->nb_threads, sizeof(*s->fft));
916 if (!s->fft)
917 return AVERROR(ENOMEM);
918
919 for (int n = 0; n < s->nb_threads; n++) {
920 ret = av_tx_init(&s->fft[n], &s->tx_fn, AV_TX_FLOAT_FFT, 0, s->input_padding_size, &scale, 0);
921 if (ret < 0)
922 return ret;
923 }
924
925 s->ifft = av_calloc(s->nb_threads, sizeof(*s->ifft));
926 if (!s->ifft)
927 return AVERROR(ENOMEM);
928
929 for (int n = 0; n < s->nb_threads; n++) {
930 ret = av_tx_init(&s->ifft[n], &s->itx_fn, AV_TX_FLOAT_FFT, 1, s->output_padding_size, &scale, 0);
931 if (ret < 0)
932 return ret;
933 }
934
935 s->outpicref = ff_get_video_buffer(outlink, outlink->w, outlink->h);
936 s->fft_in = ff_get_audio_buffer(inlink, s->fft_size * 2);
937 s->fft_out = ff_get_audio_buffer(inlink, s->fft_size * 2);
938 s->dst_x = av_frame_alloc();
939 s->src_x = av_frame_alloc();
940 s->kernel = av_calloc(s->frequency_band_count, sizeof(*s->kernel));
941 s->cache = ff_get_audio_buffer(inlink, s->hop_size);
942 s->over = ff_get_audio_buffer(inlink, s->frequency_band_count * 2 * s->ihop_size);
943 s->bh_out = ff_get_audio_buffer(inlink, s->frequency_band_count);
944 s->ifft_in = av_frame_alloc();
945 s->ifft_out = av_frame_alloc();
946 s->ch_out = av_frame_alloc();
947 s->index = av_calloc(s->input_padding_size, sizeof(*s->index));
948 s->kernel_start = av_calloc(s->frequency_band_count, sizeof(*s->kernel_start));
949 s->kernel_stop = av_calloc(s->frequency_band_count, sizeof(*s->kernel_stop));
950 if (!s->outpicref || !s->fft_in || !s->fft_out || !s->src_x || !s->dst_x || !s->over ||
951 !s->ifft_in || !s->ifft_out || !s->kernel_start || !s->kernel_stop || !s->ch_out ||
952 !s->cache || !s->index || !s->bh_out || !s->kernel)
953 return AVERROR(ENOMEM);
954
955 s->ch_out->format = inlink->format;
956 s->ch_out->nb_samples = 2 * s->ihop_size * inlink->ch_layout.nb_channels;
957 s->ch_out->ch_layout.nb_channels = s->frequency_band_count;
958 ret = av_frame_get_buffer(s->ch_out, 0);
959 if (ret < 0)
960 return ret;
961
962 s->ifft_in->format = inlink->format;
963 s->ifft_in->nb_samples = s->ifft_size * 2;
964 s->ifft_in->ch_layout.nb_channels = s->nb_threads;
965 ret = av_frame_get_buffer(s->ifft_in, 0);
966 if (ret < 0)
967 return ret;
968
969 s->ifft_out->format = inlink->format;
970 s->ifft_out->nb_samples = s->ifft_size * 2;
971 s->ifft_out->ch_layout.nb_channels = s->nb_threads;
972 ret = av_frame_get_buffer(s->ifft_out, 0);
973 if (ret < 0)
974 return ret;
975
976 s->src_x->format = inlink->format;
977 s->src_x->nb_samples = s->fft_size * 2;
978 s->src_x->ch_layout.nb_channels = s->nb_threads;
979 ret = av_frame_get_buffer(s->src_x, 0);
980 if (ret < 0)
981 return ret;
982
983 s->dst_x->format = inlink->format;
984 s->dst_x->nb_samples = s->fft_size * 2;
985 s->dst_x->ch_layout.nb_channels = s->nb_threads;
986 ret = av_frame_get_buffer(s->dst_x, 0);
987 if (ret < 0)
988 return ret;
989
990 s->outpicref->sample_aspect_ratio = (AVRational){1,1};
991
992 for (int y = 0; y < outlink->h; y++) {
993 memset(s->outpicref->data[0] + y * s->outpicref->linesize[0], 0, outlink->w);
994 memset(s->outpicref->data[1] + y * s->outpicref->linesize[1], 128, outlink->w);
995 memset(s->outpicref->data[2] + y * s->outpicref->linesize[2], 128, outlink->w);
996 if (s->outpicref->data[3])
997 memset(s->outpicref->data[3] + y * s->outpicref->linesize[3], 0, outlink->w);
998 }
999
1000 s->outpicref->color_range = AVCOL_RANGE_JPEG;
1001
1002 factor = s->input_padding_size / (float)inlink->sample_rate;
1003 for (int n = 0; n < s->frequency_band_count; n++) {
1004 s->frequency_band[2*n ] *= factor;
1005 s->frequency_band[2*n+1] *= factor;
1006 }
1007
1008 av_log(ctx, AV_LOG_DEBUG, "factor: %f\n", factor);
1009 av_log(ctx, AV_LOG_DEBUG, "nb_consumed_samples: %d\n", s->nb_consumed_samples);
1010 av_log(ctx, AV_LOG_DEBUG, "hop_size: %d\n", s->hop_size);
1011 av_log(ctx, AV_LOG_DEBUG, "ihop_size: %d\n", s->ihop_size);
1012 av_log(ctx, AV_LOG_DEBUG, "input_sample_count: %d\n", s->input_sample_count);
1013 av_log(ctx, AV_LOG_DEBUG, "input_padding_size: %d\n", s->input_padding_size);
1014 av_log(ctx, AV_LOG_DEBUG, "output_sample_count: %d\n", s->output_sample_count);
1015 av_log(ctx, AV_LOG_DEBUG, "output_padding_size: %d\n", s->output_padding_size);
1016
1017 switch (s->direction) {
1018 case DIRECTION_LR:
1019 case DIRECTION_UD:
1020 s->pos = s->bar_size;
1021 break;
1022 case DIRECTION_RL:
1023 case DIRECTION_DU:
1024 s->pos = s->sono_size;
1025 break;
1026 }
1027
1028 s->auto_frame_rate = av_make_q(inlink->sample_rate, s->hop_size);
1029 if (strcmp(s->rate_str, "auto")) {
1030 ret = av_parse_video_rate(&s->frame_rate, s->rate_str);
1031 if (ret < 0)
1032 return ret;
1033 } else {
1034 s->frame_rate = s->auto_frame_rate;
1035 }
1036 outlink->frame_rate = s->frame_rate;
1037 outlink->time_base = av_inv_q(outlink->frame_rate);
1038
1039 ret = compute_kernel(ctx);
1040 if (ret < 0)
1041 return ret;
1042
1043 return 0;
1044 }
1045
1046 static int output_frame(AVFilterContext *ctx)
1047 {
1048 AVFilterLink *outlink = ctx->outputs[0];
1049 AVFilterLink *inlink = ctx->inputs[0];
1050 ShowCWTContext *s = ctx->priv;
1051 const int nb_planes = 3 + (s->outpicref->data[3] != NULL);
1052 int ret;
1053
1054 switch (s->slide) {
1055 case SLIDE_SCROLL:
1056 switch (s->direction) {
1057 case DIRECTION_UD:
1058 for (int p = 0; p < nb_planes; p++) {
1059 ptrdiff_t linesize = s->outpicref->linesize[p];
1060
1061 for (int y = s->h - 1; y > s->bar_size; y--) {
1062 uint8_t *dst = s->outpicref->data[p] + y * linesize;
1063
1064 memmove(dst, dst - linesize, s->w);
1065 }
1066 }
1067 break;
1068 case DIRECTION_DU:
1069 for (int p = 0; p < nb_planes; p++) {
1070 ptrdiff_t linesize = s->outpicref->linesize[p];
1071
1072 for (int y = 0; y < s->sono_size; y++) {
1073 uint8_t *dst = s->outpicref->data[p] + y * linesize;
1074
1075 memmove(dst, dst + linesize, s->w);
1076 }
1077 }
1078 break;
1079 }
1080 break;
1081 }
1082
1083 ff_filter_execute(ctx, draw, NULL, NULL, s->nb_threads);
1084
1085 switch (s->slide) {
1086 case SLIDE_REPLACE:
1087 case SLIDE_FRAME:
1088 switch (s->direction) {
1089 case DIRECTION_LR:
1090 s->pos++;
1091 if (s->pos >= s->w) {
1092 s->pos = s->bar_size;
1093 s->new_frame = 1;
1094 }
1095 break;
1096 case DIRECTION_RL:
1097 s->pos--;
1098 if (s->pos < 0) {
1099 s->pos = s->sono_size;
1100 s->new_frame = 1;
1101 }
1102 break;
1103 case DIRECTION_UD:
1104 s->pos++;
1105 if (s->pos >= s->h) {
1106 s->pos = s->bar_size;
1107 s->new_frame = 1;
1108 }
1109 break;
1110 case DIRECTION_DU:
1111 s->pos--;
1112 if (s->pos < 0) {
1113 s->pos = s->sono_size;
1114 s->new_frame = 1;
1115 }
1116 break;
1117 }
1118 break;
1119 case SLIDE_SCROLL:
1120 switch (s->direction) {
1121 case DIRECTION_UD:
1122 case DIRECTION_LR:
1123 s->pos = s->bar_size;
1124 break;
1125 case DIRECTION_RL:
1126 case DIRECTION_DU:
1127 s->pos = s->sono_size;
1128 break;
1129 }
1130 break;
1131 }
1132
1133 if (s->slide == SLIDE_FRAME && s->eof) {
1134 switch (s->direction) {
1135 case DIRECTION_LR:
1136 for (int p = 0; p < nb_planes; p++) {
1137 ptrdiff_t linesize = s->outpicref->linesize[p];
1138 const int size = s->w - s->pos;
1139 const int fill = p > 0 && p < 3 ? 128 : 0;
1140 const int x = s->pos;
1141
1142 for (int y = 0; y < s->h; y++) {
1143 uint8_t *dst = s->outpicref->data[p] + y * linesize + x;
1144
1145 memset(dst, fill, size);
1146 }
1147 }
1148 break;
1149 case DIRECTION_RL:
1150 for (int p = 0; p < nb_planes; p++) {
1151 ptrdiff_t linesize = s->outpicref->linesize[p];
1152 const int size = s->w - s->pos;
1153 const int fill = p > 0 && p < 3 ? 128 : 0;
1154
1155 for (int y = 0; y < s->h; y++) {
1156 uint8_t *dst = s->outpicref->data[p] + y * linesize;
1157
1158 memset(dst, fill, size);
1159 }
1160 }
1161 break;
1162 case DIRECTION_UD:
1163 for (int p = 0; p < nb_planes; p++) {
1164 ptrdiff_t linesize = s->outpicref->linesize[p];
1165 const int fill = p > 0 && p < 3 ? 128 : 0;
1166
1167 for (int y = s->pos; y < s->h; y++) {
1168 uint8_t *dst = s->outpicref->data[p] + y * linesize;
1169
1170 memset(dst, fill, s->w);
1171 }
1172 }
1173 break;
1174 case DIRECTION_DU:
1175 for (int p = 0; p < nb_planes; p++) {
1176 ptrdiff_t linesize = s->outpicref->linesize[p];
1177 const int fill = p > 0 && p < 3 ? 128 : 0;
1178
1179 for (int y = s->h - s->pos; y >= 0; y--) {
1180 uint8_t *dst = s->outpicref->data[p] + y * linesize;
1181
1182 memset(dst, fill, s->w);
1183 }
1184 }
1185 break;
1186 }
1187 }
1188
1189 s->new_frame = s->slide == SLIDE_FRAME && (s->new_frame || s->eof);
1190
1191 if (s->slide != SLIDE_FRAME || s->new_frame == 1) {
1192 int64_t pts_offset = s->new_frame ? 0LL : av_rescale(s->ihop_index, s->hop_size, s->ihop_size);
1193 const int offset = (s->input_padding_size - s->hop_size) >> 1;
1194
1195 pts_offset = av_rescale_q(pts_offset - offset, av_make_q(1, inlink->sample_rate), inlink->time_base);
1196 s->outpicref->pts = av_rescale_q(s->in_pts + pts_offset, inlink->time_base, outlink->time_base);
1197 s->outpicref->duration = 1;
1198 }
1199
1200 s->ihop_index++;
1201 if (s->ihop_index >= s->ihop_size)
1202 s->ihop_index = s->hop_index = 0;
1203
1204 if (s->slide == SLIDE_FRAME && s->new_frame == 0)
1205 return 1;
1206
1207 if (s->old_pts < s->outpicref->pts) {
1208 AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
1209 if (!out)
1210 return AVERROR(ENOMEM);
1211 ret = av_frame_copy_props(out, s->outpicref);
1212 if (ret < 0)
1213 goto fail;
1214 ret = av_frame_copy(out, s->outpicref);
1215 if (ret < 0)
1216 goto fail;
1217 s->old_pts = s->outpicref->pts;
1218 s->new_frame = 0;
1219 ret = ff_filter_frame(outlink, out);
1220 if (ret <= 0)
1221 return ret;
1222 fail:
1223 av_frame_free(&out);
1224 return ret;
1225 }
1226
1227 return 1;
1228 }
1229
1230 static int run_channels_cwt_prepare(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
1231 {
1232 ShowCWTContext *s = ctx->priv;
1233 const int count = s->nb_channels;
1234 const int start = (count * jobnr) / nb_jobs;
1235 const int end = (count * (jobnr+1)) / nb_jobs;
1236
1237 for (int ch = start; ch < end; ch++)
1238 run_channel_cwt_prepare(ctx, arg, jobnr, ch);
1239
1240 return 0;
1241 }
1242
1243 static int activate(AVFilterContext *ctx)
1244 {
1245 AVFilterLink *inlink = ctx->inputs[0];
1246 AVFilterLink *outlink = ctx->outputs[0];
1247 ShowCWTContext *s = ctx->priv;
1248 int ret = 0, status;
1249 int64_t pts;
1250
1251 FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
1252
1253 if (s->outpicref) {
1254 AVFrame *fin = NULL;
1255
1256 if (s->hop_index < s->hop_size) {
1257 if (!s->eof) {
1258 ret = ff_inlink_consume_samples(inlink, 1, s->hop_size - s->hop_index, &fin);
1259 if (ret < 0)
1260 return ret;
1261 }
1262
1263 if (ret > 0 || s->eof) {
1264 ff_filter_execute(ctx, run_channels_cwt_prepare, fin, NULL,
1265 FFMIN(s->nb_threads, s->nb_channels));
1266 if (fin) {
1267 if (s->hop_index == 0) {
1268 s->in_pts = fin->pts;
1269 if (s->old_pts == AV_NOPTS_VALUE)
1270 s->old_pts = av_rescale_q(s->in_pts, inlink->time_base, outlink->time_base) - 1;
1271 }
1272 s->hop_index += fin->nb_samples;
1273 av_frame_free(&fin);
1274 } else {
1275 s->hop_index = s->hop_size;
1276 }
1277 }
1278 }
1279
1280 if (s->hop_index >= s->hop_size || s->ihop_index > 0) {
1281 for (int ch = 0; ch < s->nb_channels && s->ihop_index == 0; ch++) {
1282 ff_filter_execute(ctx, run_channel_cwt, (void *)&ch, NULL,
1283 s->nb_threads);
1284 }
1285
1286 ret = output_frame(ctx);
1287 if (ret != 1)
1288 return ret;
1289 }
1290 }
1291
1292 if (s->eof) {
1293 if (s->slide == SLIDE_FRAME)
1294 ret = output_frame(ctx);
1295 ff_outlink_set_status(outlink, AVERROR_EOF, s->eof_pts);
1296 return ret;
1297 }
1298
1299 if (!s->eof && ff_inlink_acknowledge_status(inlink, &status, &pts)) {
1300 if (status == AVERROR_EOF) {
1301 s->eof = 1;
1302 ff_filter_set_ready(ctx, 10);
1303 s->eof_pts = av_rescale_q(pts, inlink->time_base, outlink->time_base);
1304 return 0;
1305 }
1306 }
1307
1308 if (ff_inlink_queued_samples(inlink) > 0 || s->ihop_index ||
1309 s->hop_index >= s->hop_size || s->eof) {
1310 ff_filter_set_ready(ctx, 10);
1311 return 0;
1312 }
1313
1314 if (ff_outlink_frame_wanted(outlink)) {
1315 ff_inlink_request_frame(inlink);
1316 return 0;
1317 }
1318
1319 return FFERROR_NOT_READY;
1320 }
1321
1322 static const AVFilterPad showcwt_outputs[] = {
1323 {
1324 .name = "default",
1325 .type = AVMEDIA_TYPE_VIDEO,
1326 .config_props = config_output,
1327 },
1328 };
1329
1330 const AVFilter ff_avf_showcwt = {
1331 .name = "showcwt",
1332 .description = NULL_IF_CONFIG_SMALL("Convert input audio to a CWT (Continuous Wavelet Transform) spectrum video output."),
1333 .uninit = uninit,
1334 .priv_size = sizeof(ShowCWTContext),
1335 FILTER_INPUTS(ff_audio_default_filterpad),
1336 FILTER_OUTPUTS(showcwt_outputs),
1337 FILTER_QUERY_FUNC(query_formats),
1338 .activate = activate,
1339 .priv_class = &showcwt_class,
1340 .flags = AVFILTER_FLAG_SLICE_THREADS,
1341 };
1342