FFmpeg coverage

Directory: ../../../ffmpeg/
File: src/libavutil/tx.c
Date: 2026-08-29 16:27:19
Exec Total Coverage
Lines: 432 496 87.1%
Functions: 24 25 96.0%
Branches: 466 612 76.1%
Line Branch Exec Source
1 /*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19 #include "avassert.h"
20 #include "intmath.h"
21 #include "cpu.h"
22 #include "mem.h"
23 #include "qsort.h"
24 #include "bprint.h"
25
26 #include "tx_priv.h"
27
28 #define TYPE_IS(type, x) \
29 (((x) == AV_TX_FLOAT_ ## type) || \
30 ((x) == AV_TX_DOUBLE_ ## type) || \
31 ((x) == AV_TX_INT32_ ## type))
32
33 /* Calculates the modular multiplicative inverse */
34 5176 static av_always_inline int mulinv(int n, int m)
35 {
36 5176 n = n % m;
37
1/2
✓ Branch 0 taken 52637 times.
✗ Branch 1 not taken.
 52637 for (int x = 1; x < m; x++)
38
2/2
✓ Branch 0 taken 5176 times.
✓ Branch 1 taken 47461 times.
 52637 if (((n * x) % m) == 1)
39 5176 return x;
40 ✗ av_assert0(0); /* Never reached */
41 return 0;
42 }
43
44 81 int ff_tx_gen_pfa_input_map(AVTXContext *s, FFTXCodeletOptions *opts,
45 int d1, int d2)
46 {
47 81 const int sl = d1*d2;
48
49 81 s->map = av_malloc(s->len*sizeof(*s->map));
50
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 81 times.
 81 if (!s->map)
51 ✗ return AVERROR(ENOMEM);
52
53
2/2
✓ Branch 0 taken 81 times.
✓ Branch 1 taken 81 times.
 162 for (int k = 0; k < s->len; k += sl) {
54
3/6
✓ Branch 0 taken 81 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 81 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 81 times.
 81 if (s->inv || (opts && opts->map_dir == FF_TX_MAP_SCATTER)) {
55 ✗ for (int m = 0; m < d2; m++)
56 ✗ for (int n = 0; n < d1; n++)
57 ✗ s->map[k + ((m*d1 + n*d2) % (sl))] = m*d1 + n;
58 } else {
59
2/2
✓ Branch 0 taken 405 times.
✓ Branch 1 taken 81 times.
 486 for (int m = 0; m < d2; m++)
60
2/2
✓ Branch 0 taken 1215 times.
✓ Branch 1 taken 405 times.
 1620 for (int n = 0; n < d1; n++)
61 1215 s->map[k + m*d1 + n] = (m*d1 + n*d2) % (sl);
62 }
63
64
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 81 times.
 81 if (s->inv)
65 ✗ for (int w = 1; w <= ((sl) >> 1); w++)
66 ✗ FFSWAP(int, s->map[k + w], s->map[k + sl - w]);
67 }
68
69
1/2
✓ Branch 0 taken 81 times.
✗ Branch 1 not taken.
 81 s->map_dir = opts ? opts->map_dir : FF_TX_MAP_GATHER;
70
71 81 return 0;
72 }
73
74 /* Guaranteed to work for any n, m where gcd(n, m) == 1 */
75 2588 int ff_tx_gen_compound_mapping(AVTXContext *s, FFTXCodeletOptions *opts,
76 int inv, int n, int m)
77 {
78 int *in_map, *out_map;
79 2588 const int len = n*m; /* Will not be equal to s->len for MDCTs */
80 int m_inv, n_inv;
81
82 /* Make sure the numbers are coprime */
83
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2588 times.
 2588 if (av_gcd(n, m) != 1)
84 ✗ return AVERROR(EINVAL);
85
86 2588 m_inv = mulinv(m, n);
87 2588 n_inv = mulinv(n, m);
88
89
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2588 times.
 2588 if (!(s->map = av_malloc(2*len*sizeof(*s->map))))
90 ✗ return AVERROR(ENOMEM);
91
92 2588 in_map = s->map;
93 2588 out_map = s->map + len;
94
95 /* Ruritanian map for input, CRT map for output, can be swapped */
96
3/4
✓ Branch 0 taken 507 times.
✓ Branch 1 taken 2081 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 507 times.
 2588 if (opts && opts->map_dir == FF_TX_MAP_SCATTER) {
97 ✗ for (int j = 0; j < m; j++) {
98 ✗ for (int i = 0; i < n; i++) {
99 ✗ in_map[(i*m + j*n) % len] = j*n + i;
100 ✗ out_map[(i*m*m_inv + j*n*n_inv) % len] = i*m + j;
101 }
102 }
103 } else {
104
2/2
✓ Branch 0 taken 89332 times.
✓ Branch 1 taken 2588 times.
 91920 for (int j = 0; j < m; j++) {
105
2/2
✓ Branch 0 taken 688956 times.
✓ Branch 1 taken 89332 times.
 778288 for (int i = 0; i < n; i++) {
106 688956 in_map[j*n + i] = (i*m + j*n) % len;
107 688956 out_map[(i*m*m_inv + j*n*n_inv) % len] = i*m + j;
108 }
109 }
110 }
111
112
2/2
✓ Branch 0 taken 2480 times.
✓ Branch 1 taken 108 times.
 2588 if (inv) {
113
2/2
✓ Branch 0 taken 83500 times.
✓ Branch 1 taken 2480 times.
 85980 for (int i = 0; i < m; i++) {
114 83500 int *in = &in_map[i*n + 1]; /* Skip the DC */
115
2/2
✓ Branch 0 taken 259316 times.
✓ Branch 1 taken 83500 times.
 342816 for (int j = 0; j < ((n - 1) >> 1); j++)
116 259316 FFSWAP(int, in[j], in[n - j - 2]);
117 }
118 }
119
120
2/2
✓ Branch 0 taken 507 times.
✓ Branch 1 taken 2081 times.
 2588 s->map_dir = opts ? opts->map_dir : FF_TX_MAP_GATHER;
121
122 2588 return 0;
123 }
124
125 12398986 static inline int split_radix_permutation(int i, int len, int inv)
126 {
127 12398986 len >>= 1;
128
2/2
✓ Branch 0 taken 1612710 times.
✓ Branch 1 taken 10786276 times.
 12398986 if (len <= 1)
129 1612710 return i & 1;
130
2/2
✓ Branch 0 taken 5393138 times.
✓ Branch 1 taken 5393138 times.
 10786276 if (!(i & len))
131 5393138 return split_radix_permutation(i, len, inv) * 2;
132 5393138 len >>= 1;
133 5393138 return split_radix_permutation(i, len, inv) * 4 + 1 - 2*(!(i & len) ^ inv);
134 }
135
136 4887 int ff_tx_gen_ptwo_revtab(AVTXContext *s, FFTXCodeletOptions *opts)
137 {
138 4887 int len = s->len;
139
140
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4887 times.
 4887 if (!(s->map = av_malloc(len*sizeof(*s->map))))
141 ✗ return AVERROR(ENOMEM);
142
143
3/4
✓ Branch 0 taken 4887 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2222 times.
✓ Branch 3 taken 2665 times.
 4887 if (opts && opts->map_dir == FF_TX_MAP_SCATTER) {
144
2/2
✓ Branch 0 taken 233224 times.
✓ Branch 1 taken 2222 times.
 235446 for (int i = 0; i < s->len; i++)
145 233224 s->map[-split_radix_permutation(i, len, s->inv) & (len - 1)] = i;
146 } else {
147
2/2
✓ Branch 0 taken 1109770 times.
✓ Branch 1 taken 2665 times.
 1112435 for (int i = 0; i < s->len; i++)
148 1109770 s->map[i] = -split_radix_permutation(i, len, s->inv) & (len - 1);
149 }
150
151
1/2
✓ Branch 0 taken 4887 times.
✗ Branch 1 not taken.
 4887 s->map_dir = opts ? opts->map_dir : FF_TX_MAP_GATHER;
152
153 4887 return 0;
154 }
155
156 ✗ int ff_tx_gen_inplace_map(AVTXContext *s, int len)
157 {
158 ✗ int *src_map, out_map_idx = 0;
159
160 ✗ if (!s->sub || !s->sub->map)
161 ✗ return AVERROR(EINVAL);
162
163 ✗ if (!(s->map = av_mallocz(len*sizeof(*s->map))))
164 ✗ return AVERROR(ENOMEM);
165
166 ✗ src_map = s->sub->map;
167
168 /* The first coefficient is always already in-place */
169 ✗ for (int src = 1; src < s->len; src++) {
170 ✗ int dst = src_map[src];
171 ✗ int found = 0;
172
173 ✗ if (dst <= src)
174 ✗ continue;
175
176 /* This just checks if a closed loop has been encountered before,
177 * and if so, skips it, since to fully permute a loop we must only
178 * enter it once. */
179 do {
180 ✗ for (int j = 0; j < out_map_idx; j++) {
181 ✗ if (dst == s->map[j]) {
182 ✗ found = 1;
183 ✗ break;
184 }
185 }
186 ✗ dst = src_map[dst];
187 ✗ } while (dst != src && !found);
188
189 ✗ if (!found)
190 ✗ s->map[out_map_idx++] = src;
191 }
192
193 ✗ s->map[out_map_idx++] = 0;
194
195 ✗ return 0;
196 }
197
198 66491 static void parity_revtab_generator(int *revtab, int n, int inv, int offset,
199 int is_dual, int dual_high, int len,
200 int basis, int dual_stride, int inv_lookup)
201 {
202 66491 len >>= 1;
203
204
2/2
✓ Branch 0 taken 44883 times.
✓ Branch 1 taken 21608 times.
 66491 if (len <= basis) {
205 int k1, k2, stride, even_idx, odd_idx;
206
207
3/4
✓ Branch 0 taken 33530 times.
✓ Branch 1 taken 11353 times.
✓ Branch 2 taken 33530 times.
✗ Branch 3 not taken.
 44883 is_dual = is_dual && dual_stride;
208 44883 dual_high = is_dual & dual_high;
209
2/2
✓ Branch 0 taken 33530 times.
✓ Branch 1 taken 11353 times.
 44883 stride = is_dual ? FFMIN(dual_stride, len) : 0;
210
211 44883 even_idx = offset + dual_high*(stride - 2*len);
212
4/4
✓ Branch 0 taken 33530 times.
✓ Branch 1 taken 11353 times.
✓ Branch 2 taken 16765 times.
✓ Branch 3 taken 16765 times.
 44883 odd_idx = even_idx + len + (is_dual && !dual_high)*len + dual_high*len;
213
214
2/2
✓ Branch 0 taken 134858 times.
✓ Branch 1 taken 44883 times.
 179741 for (int i = 0; i < len; i++) {
215 134858 k1 = -split_radix_permutation(offset + i*2 + 0, n, inv) & (n - 1);
216 134858 k2 = -split_radix_permutation(offset + i*2 + 1, n, inv) & (n - 1);
217
2/2
✓ Branch 0 taken 100692 times.
✓ Branch 1 taken 34166 times.
 134858 if (inv_lookup) {
218 100692 revtab[even_idx++] = k1;
219 100692 revtab[odd_idx++] = k2;
220 } else {
221 34166 revtab[k1] = even_idx++;
222 34166 revtab[k2] = odd_idx++;
223 }
224
4/4
✓ Branch 0 taken 89780 times.
✓ Branch 1 taken 45078 times.
✓ Branch 2 taken 44890 times.
✓ Branch 3 taken 44890 times.
 134858 if (stride && !((i + 1) % stride)) {
225 44890 even_idx += stride;
226 44890 odd_idx += stride;
227 }
228 }
229
230 44883 return;
231 }
232
233 21608 parity_revtab_generator(revtab, n, inv, offset,
234 0, 0, len >> 0, basis, dual_stride, inv_lookup);
235 21608 parity_revtab_generator(revtab, n, inv, offset + (len >> 0),
236 1, 0, len >> 1, basis, dual_stride, inv_lookup);
237 21608 parity_revtab_generator(revtab, n, inv, offset + (len >> 0) + (len >> 1),
238 1, 1, len >> 1, basis, dual_stride, inv_lookup);
239 }
240
241 1667 int ff_tx_gen_split_radix_parity_revtab(AVTXContext *s, int len, int inv,
242 FFTXCodeletOptions *opts,
243 int basis, int dual_stride)
244 {
245 1667 basis >>= 1;
246
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1667 times.
 1667 if (len < basis)
247 ✗ return AVERROR(EINVAL);
248
249
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1667 times.
 1667 if (!(s->map = av_mallocz(len*sizeof(*s->map))))
250 ✗ return AVERROR(ENOMEM);
251
252
3/4
✓ Branch 0 taken 1399 times.
✓ Branch 1 taken 268 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1399 times.
 1667 av_assert0(!dual_stride || !(dual_stride & (dual_stride - 1)));
253
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1667 times.
 1667 av_assert0(dual_stride <= basis);
254
255
2/2
✓ Branch 0 taken 1643 times.
✓ Branch 1 taken 24 times.
 3310 parity_revtab_generator(s->map, len, inv, 0, 0, 0, len,
256 basis, dual_stride,
257
2/2
✓ Branch 0 taken 833 times.
✓ Branch 1 taken 810 times.
 1643 opts ? opts->map_dir == FF_TX_MAP_GATHER : FF_TX_MAP_GATHER);
258
259
2/2
✓ Branch 0 taken 1643 times.
✓ Branch 1 taken 24 times.
 1667 s->map_dir = opts ? opts->map_dir : FF_TX_MAP_GATHER;
260
261 1667 return 0;
262 }
263
264 40573 static void reset_ctx(AVTXContext *s, int free_sub)
265 {
266
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40573 times.
 40573 if (!s)
267 ✗ return;
268
269
2/2
✓ Branch 0 taken 7254 times.
✓ Branch 1 taken 33319 times.
 40573 if (s->sub)
270
2/2
✓ Branch 0 taken 29016 times.
✓ Branch 1 taken 7254 times.
 36270 for (int i = 0; i < TX_MAX_SUB; i++)
271 29016 reset_ctx(&s->sub[i], free_sub + 1);
272
273
3/4
✓ Branch 0 taken 18908 times.
✓ Branch 1 taken 21665 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 18908 times.
 40573 if (s->cd_self && s->cd_self->uninit)
274 ✗ s->cd_self->uninit(s);
275
276
2/2
✓ Branch 0 taken 35619 times.
✓ Branch 1 taken 4954 times.
 40573 if (free_sub)
277 35619 av_freep(&s->sub);
278
279 40573 av_freep(&s->map);
280 40573 av_freep(&s->exp);
281 40573 av_freep(&s->tmp);
282
283 /* Nothing else needs to be reset, it gets overwritten if another
284 * ff_tx_init_subtx() call is made. */
285 40573 s->nb_sub = 0;
286 40573 s->opaque = NULL;
287 40573 memset(s->fn, 0, sizeof(s->fn));
288 }
289
290 97 void ff_tx_clear_ctx(AVTXContext *s)
291 {
292 97 reset_ctx(s, 0);
293 97 }
294
295 7252 av_cold void av_tx_uninit(AVTXContext **ctx)
296 {
297
2/2
✓ Branch 0 taken 649 times.
✓ Branch 1 taken 6603 times.
 7252 if (!(*ctx))
298 649 return;
299
300 6603 reset_ctx(*ctx, 1);
301 6603 av_freep(ctx);
302 }
303
304 14 static av_cold int ff_tx_null_init(AVTXContext *s, const FFTXCodelet *cd,
305 uint64_t flags, FFTXCodeletOptions *opts,
306 int len, int inv, const void *scale)
307 {
308 /* Can only handle one sample+type to one sample+type transforms */
309
6/12
✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 14 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 14 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 14 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 14 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 14 times.
 14 if (TYPE_IS(MDCT, s->type) || TYPE_IS(RDFT, s->type))
310 ✗ return AVERROR(EINVAL);
311 14 return 0;
312 }
313
314 /* Null transform when the length is 1 */
315 3 static void ff_tx_null(AVTXContext *s, void *_out, void *_in, ptrdiff_t stride)
316 {
317 3 memcpy(_out, _in, stride);
318 3 }
319
320 static const FFTXCodelet ff_tx_null_def = {
321 .name = NULL_IF_CONFIG_SMALL("null"),
322 .function = ff_tx_null,
323 .type = TX_TYPE_ANY,
324 .flags = AV_TX_UNALIGNED | FF_TX_ALIGNED |
325 FF_TX_OUT_OF_PLACE | AV_TX_INPLACE,
326 .factors[0] = TX_FACTOR_ANY,
327 .min_len = 1,
328 .max_len = 1,
329 .init = ff_tx_null_init,
330 .cpu_flags = FF_TX_CPU_FLAGS_ALL,
331 .prio = FF_TX_PRIO_MAX,
332 };
333
334 static const FFTXCodelet * const ff_tx_null_list[] = {
335 &ff_tx_null_def,
336 NULL,
337 };
338
339 /* Array of all compiled codelet lists. Order is irrelevant. */
340 static const FFTXCodelet * const * const codelet_list[] = {
341 ff_tx_codelet_list_float_c,
342 ff_tx_codelet_list_double_c,
343 ff_tx_codelet_list_int32_c,
344 ff_tx_null_list,
345 #if HAVE_X86ASM
346 ff_tx_codelet_list_float_x86,
347 #endif
348 #if ARCH_AARCH64
349 ff_tx_codelet_list_float_aarch64,
350 #endif
351 };
352 static const int codelet_list_num = FF_ARRAY_ELEMS(codelet_list);
353
354 static const int cpu_slow_mask = AV_CPU_FLAG_SSE2SLOW | AV_CPU_FLAG_SSE3SLOW |
355 AV_CPU_FLAG_ATOM | AV_CPU_FLAG_SSSE3SLOW |
356 AV_CPU_FLAG_AVXSLOW | AV_CPU_FLAG_SLOW_GATHER;
357
358 static const int cpu_slow_penalties[][2] = {
359 { AV_CPU_FLAG_SSE2SLOW, 1 + 64 },
360 { AV_CPU_FLAG_SSE3SLOW, 1 + 64 },
361 { AV_CPU_FLAG_SSSE3SLOW, 1 + 64 },
362 { AV_CPU_FLAG_ATOM, 1 + 128 },
363 { AV_CPU_FLAG_AVXSLOW, 1 + 128 },
364 { AV_CPU_FLAG_SLOW_GATHER, 1 + 32 },
365 };
366
367 37300 static int get_codelet_prio(const FFTXCodelet *cd, int cpu_flags, int len)
368 {
369 37300 int prio = cd->prio;
370 37300 int max_factor = 0;
371
372 /* If the CPU has a SLOW flag, and the instruction is also flagged
373 * as being slow for such, reduce its priority */
374
2/2
✓ Branch 0 taken 223800 times.
✓ Branch 1 taken 37300 times.
 261100 for (int i = 0; i < FF_ARRAY_ELEMS(cpu_slow_penalties); i++) {
375
2/2
✓ Branch 0 taken 2580 times.
✓ Branch 1 taken 221220 times.
 223800 if ((cpu_flags & cd->cpu_flags) & cpu_slow_penalties[i][0])
376 2580 prio -= cpu_slow_penalties[i][1];
377 }
378
379 /* Prioritize aligned-only codelets */
380
4/4
✓ Branch 0 taken 5800 times.
✓ Branch 1 taken 31500 times.
✓ Branch 2 taken 5786 times.
✓ Branch 3 taken 14 times.
 37300 if ((cd->flags & FF_TX_ALIGNED) && !(cd->flags & AV_TX_UNALIGNED))
381 5786 prio += 64;
382
383 /* Codelets for specific lengths are generally faster */
384
4/4
✓ Branch 0 taken 8644 times.
✓ Branch 1 taken 28656 times.
✓ Branch 2 taken 7854 times.
✓ Branch 3 taken 790 times.
 37300 if ((len == cd->min_len) && (len == cd->max_len))
385 7854 prio += 64;
386
387 /* Forward-only or inverse-only transforms are generally better */
388
2/2
✓ Branch 0 taken 20135 times.
✓ Branch 1 taken 17165 times.
 37300 if ((cd->flags & (FF_TX_FORWARD_ONLY | FF_TX_INVERSE_ONLY)))
389 20135 prio += 64;
390
391 /* Larger factors are generally better */
392
2/2
✓ Branch 0 taken 149200 times.
✓ Branch 1 taken 37300 times.
 186500 for (int i = 0; i < TX_MAX_SUB; i++)
393 149200 max_factor = FFMAX(cd->factors[i], max_factor);
394
2/2
✓ Branch 0 taken 35390 times.
✓ Branch 1 taken 1910 times.
 37300 if (max_factor)
395 35390 prio += 16*max_factor;
396
397 37300 return prio;
398 }
399
400 typedef struct FFTXLenDecomp {
401 int len;
402 int len2;
403 int prio;
404 const FFTXCodelet *cd;
405 } FFTXLenDecomp;
406
407 475 static int cmp_decomp(FFTXLenDecomp *a, FFTXLenDecomp *b)
408 {
409 475 return FFDIFFSIGN(b->prio, a->prio);
410 }
411
412 4762 int ff_tx_decompose_length(int dst[TX_MAX_DECOMPOSITIONS], enum AVTXType type,
413 int len, int inv)
414 {
415 4762 int nb_decomp = 0;
416 FFTXLenDecomp ld[TX_MAX_DECOMPOSITIONS];
417 4762 int codelet_list_idx = codelet_list_num;
418
419 4762 const int cpu_flags = av_get_cpu_flags();
420
421 /* Loop through all codelets in all codelet lists to find matches
422 * to the requirements */
423
2/2
✓ Branch 0 taken 23810 times.
✓ Branch 1 taken 4762 times.
 28572 while (codelet_list_idx--) {
424 23810 const FFTXCodelet * const * list = codelet_list[codelet_list_idx];
425 23810 const FFTXCodelet *cd = NULL;
426
427
2/2
✓ Branch 0 taken 1028592 times.
✓ Branch 1 taken 23810 times.
 1052402 while ((cd = *list++)) {
428 1028592 int fl = len;
429 1028592 int skip = 0, prio;
430 1028592 int factors_product = 1, factors_mod = 0;
431
432
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1028592 times.
 1028592 if (nb_decomp >= TX_MAX_DECOMPOSITIONS)
433 ✗ goto sort;
434
435 /* Check if the type matches */
436
4/4
✓ Branch 0 taken 1023830 times.
✓ Branch 1 taken 4762 times.
✓ Branch 2 taken 697404 times.
✓ Branch 3 taken 326426 times.
 1028592 if (cd->type != TX_TYPE_ANY && type != cd->type)
437 697404 continue;
438
439 /* Check direction for non-orthogonal codelets */
440
4/4
✓ Branch 0 taken 23280 times.
✓ Branch 1 taken 307908 times.
✓ Branch 2 taken 4038 times.
✓ Branch 3 taken 19242 times.
 331188 if (((cd->flags & FF_TX_FORWARD_ONLY) && inv) ||
441
4/4
✓ Branch 0 taken 8464 times.
✓ Branch 1 taken 303482 times.
✓ Branch 2 taken 7308 times.
✓ Branch 3 taken 1156 times.
 311946 ((cd->flags & (FF_TX_INVERSE_ONLY | AV_TX_FULL_IMDCT)) && !inv) ||
442
3/4
✓ Branch 0 taken 4038 times.
✓ Branch 1 taken 306752 times.
✓ Branch 2 taken 4038 times.
✗ Branch 3 not taken.
 310790 ((cd->flags & (FF_TX_FORWARD_ONLY | AV_TX_REAL_TO_REAL)) && inv) ||
443
3/4
✓ Branch 0 taken 4038 times.
✓ Branch 1 taken 306752 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 4038 times.
 310790 ((cd->flags & (FF_TX_FORWARD_ONLY | AV_TX_REAL_TO_IMAGINARY)) && inv))
444 20398 continue;
445
446 /* Check if the CPU supports the required ISA */
447
2/2
✓ Branch 0 taken 164470 times.
✓ Branch 1 taken 146320 times.
 310790 if (cd->cpu_flags != FF_TX_CPU_FLAGS_ALL &&
448
2/2
✓ Branch 0 taken 163062 times.
✓ Branch 1 taken 1408 times.
 164470 !(cpu_flags & (cd->cpu_flags & ~cpu_slow_mask)))
449 163062 continue;
450
451
1/2
✓ Branch 0 taken 324028 times.
✗ Branch 1 not taken.
 324028 for (int i = 0; i < TX_MAX_FACTORS; i++) {
452
4/4
✓ Branch 0 taken 185808 times.
✓ Branch 1 taken 138220 times.
✓ Branch 2 taken 176300 times.
✓ Branch 3 taken 9508 times.
 324028 if (!cd->factors[i] || (fl == 1))
453 break;
454
455
2/2
✓ Branch 0 taken 28588 times.
✓ Branch 1 taken 147712 times.
 176300 if (cd->factors[i] == TX_FACTOR_ANY) {
456 28588 factors_mod++;
457 28588 factors_product *= fl;
458
2/2
✓ Branch 0 taken 92423 times.
✓ Branch 1 taken 55289 times.
 147712 } else if (!(fl % cd->factors[i])) {
459 92423 factors_mod++;
460
2/2
✓ Branch 0 taken 91582 times.
✓ Branch 1 taken 841 times.
 92423 if (cd->factors[i] == 2) {
461 91582 int b = ff_ctz(fl);
462 91582 fl >>= b;
463 91582 factors_product <<= b;
464 } else {
465 do {
466 873 fl /= cd->factors[i];
467 873 factors_product *= cd->factors[i];
468
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 841 times.
 873 } while (!(fl % cd->factors[i]));
469 }
470 }
471 }
472
473 /* Disqualify if factor requirements are not satisfied or if trivial */
474
4/4
✓ Branch 0 taken 111325 times.
✓ Branch 1 taken 36403 times.
✓ Branch 2 taken 109155 times.
✓ Branch 3 taken 2170 times.
 147728 if ((factors_mod < cd->nb_factors) || (len == factors_product))
475 145558 continue;
476
477
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2170 times.
 2170 if (av_gcd(factors_product, fl) != 1)
478 ✗ continue;
479
480 /* Check if length is supported and factorization was successful */
481
2/2
✓ Branch 0 taken 1198 times.
✓ Branch 1 taken 972 times.
 2170 if ((factors_product < cd->min_len) ||
482
3/4
✓ Branch 0 taken 1198 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 616 times.
✓ Branch 3 taken 582 times.
 1198 (cd->max_len != TX_LEN_UNLIMITED && (factors_product > cd->max_len)))
483 1588 continue;
484
485 582 prio = get_codelet_prio(cd, cpu_flags, factors_product) * factors_product;
486
487 /* Check for duplicates */
488
2/2
✓ Branch 0 taken 979 times.
✓ Branch 1 taken 348 times.
 1327 for (int i = 0; i < nb_decomp; i++) {
489
2/2
✓ Branch 0 taken 234 times.
✓ Branch 1 taken 745 times.
 979 if (factors_product == ld[i].len) {
490 /* Update priority if new one is higher */
491
2/2
✓ Branch 0 taken 204 times.
✓ Branch 1 taken 30 times.
 234 if (prio > ld[i].prio)
492 204 ld[i].prio = prio;
493 234 skip = 1;
494 234 break;
495 }
496 }
497
498 /* Add decomposition if unique */
499
2/2
✓ Branch 0 taken 348 times.
✓ Branch 1 taken 234 times.
 582 if (!skip) {
500 348 ld[nb_decomp].cd = cd;
501 348 ld[nb_decomp].len = factors_product;
502 348 ld[nb_decomp].len2 = fl;
503 348 ld[nb_decomp].prio = prio;
504 348 nb_decomp++;
505 }
506 }
507 }
508
509
2/2
✓ Branch 0 taken 4665 times.
✓ Branch 1 taken 97 times.
 4762 if (!nb_decomp)
510 4665 return AVERROR(EINVAL);
511
512 97 sort:
513
27/44
✓ Branch 0 taken 81 times.
✓ Branch 1 taken 89 times.
✓ Branch 3 taken 27 times.
✓ Branch 4 taken 54 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 27 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 54 times.
✓ Branch 12 taken 81 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 81 times.
✓ Branch 16 taken 81 times.
✓ Branch 17 taken 19 times.
✓ Branch 19 taken 19 times.
✓ Branch 20 taken 62 times.
✓ Branch 21 taken 62 times.
✓ Branch 22 taken 81 times.
✓ Branch 24 taken 62 times.
✗ Branch 25 not taken.
✗ Branch 26 not taken.
✓ Branch 27 taken 81 times.
✓ Branch 28 taken 81 times.
✓ Branch 29 taken 81 times.
✗ Branch 30 not taken.
✓ Branch 31 taken 81 times.
✗ Branch 32 not taken.
✗ Branch 33 not taken.
✗ Branch 34 not taken.
✗ Branch 35 not taken.
✗ Branch 36 not taken.
✗ Branch 37 not taken.
✗ Branch 39 not taken.
✗ Branch 40 not taken.
✗ Branch 41 not taken.
✗ Branch 42 not taken.
✓ Branch 43 taken 19 times.
✓ Branch 44 taken 62 times.
✓ Branch 46 taken 8 times.
✓ Branch 47 taken 81 times.
✓ Branch 48 taken 170 times.
✓ Branch 49 taken 89 times.
✓ Branch 50 taken 178 times.
✓ Branch 51 taken 97 times.
 599 AV_QSORT(ld, nb_decomp, FFTXLenDecomp, cmp_decomp);
514
515
2/2
✓ Branch 0 taken 348 times.
✓ Branch 1 taken 97 times.
 445 for (int i = 0; i < nb_decomp; i++) {
516
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 348 times.
 348 if (ld[i].cd->nb_factors > 1)
517 ✗ dst[i] = ld[i].len2;
518 else
519 348 dst[i] = ld[i].len;
520 }
521
522 97 return nb_decomp;
523 }
524
525 24 int ff_tx_gen_default_map(AVTXContext *s, FFTXCodeletOptions *opts)
526 {
527 24 s->map = av_malloc(s->len*sizeof(*s->map));
528
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 if (!s->map)
529 ✗ return AVERROR(ENOMEM);
530
531 24 s->map[0] = 0; /* DC is always at the start */
532
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 if (s->inv) /* Reversing the ACs flips the transform direction */
533 ✗ for (int i = 1; i < s->len; i++)
534 ✗ s->map[i] = s->len - i;
535 else
536
2/2
✓ Branch 0 taken 144 times.
✓ Branch 1 taken 24 times.
 168 for (int i = 1; i < s->len; i++)
537 144 s->map[i] = i;
538
539 24 s->map_dir = FF_TX_MAP_GATHER;
540
541 24 return 0;
542 }
543
544 #if !CONFIG_SMALL
545 64840 static void print_flags(AVBPrint *bp, uint64_t f)
546 {
547 64840 int prev = 0;
548 64840 const char *sep = ", ";
549 64840 av_bprintf(bp, "flags: [");
550
3/4
✓ Branch 0 taken 23453 times.
✓ Branch 1 taken 41387 times.
✓ Branch 2 taken 23453 times.
✗ Branch 3 not taken.
 64840 if ((f & FF_TX_ALIGNED) && ++prev)
551 23453 av_bprintf(bp, "aligned");
552
3/4
✓ Branch 0 taken 41415 times.
✓ Branch 1 taken 23425 times.
✓ Branch 2 taken 41415 times.
✗ Branch 3 not taken.
 64840 if ((f & AV_TX_UNALIGNED) && ++prev)
553
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 41387 times.
 41415 av_bprintf(bp, "%sunaligned", prev > 1 ? sep : "");
554
3/4
✓ Branch 0 taken 30474 times.
✓ Branch 1 taken 34366 times.
✓ Branch 2 taken 30474 times.
✗ Branch 3 not taken.
 64840 if ((f & AV_TX_INPLACE) && ++prev)
555
1/2
✓ Branch 0 taken 30474 times.
✗ Branch 1 not taken.
 30474 av_bprintf(bp, "%sinplace", prev > 1 ? sep : "");
556
3/4
✓ Branch 0 taken 57567 times.
✓ Branch 1 taken 7273 times.
✓ Branch 2 taken 57567 times.
✗ Branch 3 not taken.
 64840 if ((f & FF_TX_OUT_OF_PLACE) && ++prev)
557
1/2
✓ Branch 0 taken 57567 times.
✗ Branch 1 not taken.
 57567 av_bprintf(bp, "%sout_of_place", prev > 1 ? sep : "");
558
3/4
✓ Branch 0 taken 1516 times.
✓ Branch 1 taken 63324 times.
✓ Branch 2 taken 1516 times.
✗ Branch 3 not taken.
 64840 if ((f & FF_TX_FORWARD_ONLY) && ++prev)
559
1/2
✓ Branch 0 taken 1516 times.
✗ Branch 1 not taken.
 1516 av_bprintf(bp, "%sfwd_only", prev > 1 ? sep : "");
560
3/4
✓ Branch 0 taken 24728 times.
✓ Branch 1 taken 40112 times.
✓ Branch 2 taken 24728 times.
✗ Branch 3 not taken.
 64840 if ((f & FF_TX_INVERSE_ONLY) && ++prev)
561
1/2
✓ Branch 0 taken 24728 times.
✗ Branch 1 not taken.
 24728 av_bprintf(bp, "%sinv_only", prev > 1 ? sep : "");
562
3/4
✓ Branch 0 taken 28856 times.
✓ Branch 1 taken 35984 times.
✓ Branch 2 taken 28856 times.
✗ Branch 3 not taken.
 64840 if ((f & FF_TX_PRESHUFFLE) && ++prev)
563
1/2
✓ Branch 0 taken 28856 times.
✗ Branch 1 not taken.
 28856 av_bprintf(bp, "%spreshuf", prev > 1 ? sep : "");
564
3/4
✓ Branch 0 taken 306 times.
✓ Branch 1 taken 64534 times.
✓ Branch 2 taken 306 times.
✗ Branch 3 not taken.
 64840 if ((f & AV_TX_FULL_IMDCT) && ++prev)
565
1/2
✓ Branch 0 taken 306 times.
✗ Branch 1 not taken.
 306 av_bprintf(bp, "%simdct_full", prev > 1 ? sep : "");
566
3/4
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 64816 times.
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
 64840 if ((f & AV_TX_REAL_TO_REAL) && ++prev)
567
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 av_bprintf(bp, "%sreal_to_real", prev > 1 ? sep : "");
568
3/4
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 64816 times.
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
 64840 if ((f & AV_TX_REAL_TO_IMAGINARY) && ++prev)
569
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 av_bprintf(bp, "%sreal_to_imaginary", prev > 1 ? sep : "");
570
3/4
✓ Branch 0 taken 7212 times.
✓ Branch 1 taken 57628 times.
✓ Branch 2 taken 7212 times.
✗ Branch 3 not taken.
 64840 if ((f & FF_TX_ASM_CALL) && ++prev)
571
1/2
✓ Branch 0 taken 7212 times.
✗ Branch 1 not taken.
 7212 av_bprintf(bp, "%sasm_call", prev > 1 ? sep : "");
572 64840 av_bprintf(bp, "]");
573 64840 }
574
575 64840 static void print_type(AVBPrint *bp, enum AVTXType type)
576 {
577
2/2
✓ Branch 0 taken 64812 times.
✓ Branch 1 taken 28 times.
 129652 av_bprintf(bp, "%s",
578 type == TX_TYPE_ANY ? "any" :
579
2/2
✓ Branch 0 taken 35970 times.
✓ Branch 1 taken 28842 times.
 100782 type == AV_TX_FLOAT_FFT ? "fft_float" :
580
2/2
✓ Branch 0 taken 4789 times.
✓ Branch 1 taken 31181 times.
 40759 type == AV_TX_FLOAT_MDCT ? "mdct_float" :
581
2/2
✓ Branch 0 taken 4594 times.
✓ Branch 1 taken 195 times.
 9383 type == AV_TX_FLOAT_RDFT ? "rdft_float" :
582
2/2
✓ Branch 0 taken 4570 times.
✓ Branch 1 taken 24 times.
 9164 type == AV_TX_FLOAT_DCT_I ? "dctI_float" :
583
2/2
✓ Branch 0 taken 4546 times.
✓ Branch 1 taken 24 times.
 9116 type == AV_TX_FLOAT_DST_I ? "dstI_float" :
584
2/2
✓ Branch 0 taken 2614 times.
✓ Branch 1 taken 1932 times.
 7160 type == AV_TX_DOUBLE_FFT ? "fft_double" :
585
1/2
✓ Branch 0 taken 2614 times.
✗ Branch 1 not taken.
 5228 type == AV_TX_DOUBLE_MDCT ? "mdct_double" :
586
1/2
✓ Branch 0 taken 2614 times.
✗ Branch 1 not taken.
 5228 type == AV_TX_DOUBLE_RDFT ? "rdft_double" :
587
1/2
✓ Branch 0 taken 2614 times.
✗ Branch 1 not taken.
 5228 type == AV_TX_DOUBLE_DCT_I ? "dctI_double" :
588
1/2
✓ Branch 0 taken 2614 times.
✗ Branch 1 not taken.
 5228 type == AV_TX_DOUBLE_DST_I ? "dstI_double" :
589
2/2
✓ Branch 0 taken 1439 times.
✓ Branch 1 taken 1175 times.
 4053 type == AV_TX_INT32_FFT ? "fft_int32" :
590
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 1427 times.
 1451 type == AV_TX_INT32_MDCT ? "mdct_int32" :
591
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 24 type == AV_TX_INT32_RDFT ? "rdft_int32" :
592
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 24 type == AV_TX_INT32_DCT_I ? "dctI_int32" :
593
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12 times.
 12 type == AV_TX_INT32_DST_I ? "dstI_int32" :
594 "unknown");
595 64840 }
596
597 50672 static void print_cd_info(const FFTXCodelet *cd, int prio, int len, int print_prio,
598 int log_level)
599 {
600 AVBPrint bp;
601 50672 av_bprint_init(&bp, 0, AV_BPRINT_SIZE_AUTOMATIC);
602
603 50672 av_bprintf(&bp, "%s - type: ", cd->name);
604
605 50672 print_type(&bp, cd->type);
606
607 50672 av_bprintf(&bp, ", len: ");
608
2/2
✓ Branch 0 taken 36718 times.
✓ Branch 1 taken 13954 times.
 50672 if (!len) {
609
2/2
✓ Branch 0 taken 29428 times.
✓ Branch 1 taken 7290 times.
 36718 if (cd->min_len != cd->max_len)
610 29428 av_bprintf(&bp, "[%i, ", cd->min_len);
611
612
2/2
✓ Branch 0 taken 26929 times.
✓ Branch 1 taken 9789 times.
 36718 if (cd->max_len == TX_LEN_UNLIMITED)
613 26929 av_bprintf(&bp, "∞");
614 else
615 9789 av_bprintf(&bp, "%i", cd->max_len);
616 } else {
617 13954 av_bprintf(&bp, "%i", len);
618 }
619
620
2/2
✓ Branch 0 taken 32754 times.
✓ Branch 1 taken 17918 times.
 50672 if (cd->factors[1]) {
621
3/4
✓ Branch 0 taken 25744 times.
✓ Branch 1 taken 7010 times.
✓ Branch 2 taken 25744 times.
✗ Branch 3 not taken.
 32754 av_bprintf(&bp, "%s, factors", !len && cd->min_len != cd->max_len ? "]" : "");
622
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 32746 times.
 32754 if (!cd->nb_factors)
623 8 av_bprintf(&bp, ": [");
624 else
625 32746 av_bprintf(&bp, "[%i]: [", cd->nb_factors);
626
627
1/2
✓ Branch 0 taken 114084 times.
✗ Branch 1 not taken.
 114084 for (int i = 0; i < TX_MAX_FACTORS; i++) {
628
4/4
✓ Branch 0 taken 81330 times.
✓ Branch 1 taken 32754 times.
✓ Branch 2 taken 48576 times.
✓ Branch 3 taken 32754 times.
 114084 if (i && cd->factors[i])
629 48576 av_bprintf(&bp, ", ");
630
2/2
✓ Branch 0 taken 31734 times.
✓ Branch 1 taken 82350 times.
 114084 if (cd->factors[i] == TX_FACTOR_ANY)
631 31734 av_bprintf(&bp, "any");
632
2/2
✓ Branch 0 taken 49596 times.
✓ Branch 1 taken 32754 times.
 82350 else if (cd->factors[i])
633 49596 av_bprintf(&bp, "%i", cd->factors[i]);
634 else
635 32754 break;
636 }
637
638 32754 av_bprintf(&bp, "], ");
639 } else {
640 17918 av_bprintf(&bp, "%s, factor: %i, ",
641
4/4
✓ Branch 0 taken 10974 times.
✓ Branch 1 taken 6944 times.
✓ Branch 2 taken 3684 times.
✓ Branch 3 taken 7290 times.
 17918 !len && cd->min_len != cd->max_len ? "]" : "", cd->factors[0]);
642 }
643 50672 print_flags(&bp, cd->flags);
644
645
2/2
✓ Branch 0 taken 36718 times.
✓ Branch 1 taken 13954 times.
 50672 if (print_prio)
646 36718 av_bprintf(&bp, ", prio: %i", prio);
647
648 50672 av_log(NULL, log_level, "%s\n", bp.str);
649 50672 }
650
651 13954 static void print_tx_structure(AVTXContext *s, int depth)
652 {
653 13954 const FFTXCodelet *cd = s->cd_self;
654
655
2/2
✓ Branch 0 taken 22051 times.
✓ Branch 1 taken 13954 times.
 36005 for (int i = 0; i <= depth; i++)
656 22051 av_log(NULL, AV_LOG_DEBUG, " ");
657
658 13954 print_cd_info(cd, cd->prio, s->len, 0, AV_LOG_DEBUG);
659
660
2/2
✓ Branch 0 taken 7351 times.
✓ Branch 1 taken 13954 times.
 21305 for (int i = 0; i < s->nb_sub; i++)
661 7351 print_tx_structure(&s->sub[i], depth + 1);
662 13954 }
663 #endif /* CONFIG_SMALL */
664
665 typedef struct TXCodeletMatch {
666 const FFTXCodelet *cd;
667 int prio;
668 } TXCodeletMatch;
669
670 37340 static int cmp_matches(TXCodeletMatch *a, TXCodeletMatch *b)
671 {
672 37340 return FFDIFFSIGN(b->prio, a->prio);
673 }
674
675 /* We want all factors to completely cover the length */
676 63665 static inline int check_cd_factors(const FFTXCodelet *cd, int len)
677 {
678 63665 int matches = 0, any_flag = 0;
679
680
1/2
✓ Branch 0 taken 194147 times.
✗ Branch 1 not taken.
 194147 for (int i = 0; i < TX_MAX_FACTORS; i++) {
681 194147 int factor = cd->factors[i];
682
683
2/2
✓ Branch 0 taken 51967 times.
✓ Branch 1 taken 142180 times.
 194147 if (factor == TX_FACTOR_ANY) {
684 51967 any_flag = 1;
685 51967 matches++;
686 51967 continue;
687
4/4
✓ Branch 0 taken 118415 times.
✓ Branch 1 taken 23765 times.
✓ Branch 2 taken 78515 times.
✓ Branch 3 taken 39900 times.
 142180 } else if (len <= 1 || !factor) {
688 break;
689
2/2
✓ Branch 0 taken 30668 times.
✓ Branch 1 taken 47847 times.
 78515 } else if (factor == 2) { /* Fast path */
690 30668 int bits_2 = ff_ctz(len);
691
2/2
✓ Branch 0 taken 48 times.
✓ Branch 1 taken 30620 times.
 30668 if (!bits_2)
692 48 continue; /* Factor not supported */
693
694 30620 len >>= bits_2;
695 30620 matches++;
696 } else {
697 47847 int res = len % factor;
698
2/2
✓ Branch 0 taken 40857 times.
✓ Branch 1 taken 6990 times.
 47847 if (res)
699 40857 continue; /* Factor not supported */
700
701
2/2
✓ Branch 0 taken 7200 times.
✓ Branch 1 taken 6990 times.
 14190 while (!res) {
702 7200 len /= factor;
703 7200 res = len % factor;
704 }
705 6990 matches++;
706 }
707 }
708
709
6/6
✓ Branch 0 taken 38065 times.
✓ Branch 1 taken 25600 times.
✓ Branch 2 taken 11010 times.
✓ Branch 3 taken 27055 times.
✓ Branch 4 taken 9663 times.
✓ Branch 5 taken 1347 times.
 63665 return (cd->nb_factors <= matches) && (any_flag || len == 1);
710 }
711
712 14168 av_cold int ff_tx_init_subtx(AVTXContext *s, enum AVTXType type,
713 uint64_t flags, FFTXCodeletOptions *opts,
714 int len, int inv, const void *scale)
715 {
716 14168 int ret = 0;
717 14168 AVTXContext *sub = NULL;
718 14168 TXCodeletMatch *cd_tmp, *cd_matches = NULL;
719 14168 unsigned int cd_matches_size = 0;
720 14168 int codelet_list_idx = codelet_list_num;
721 14168 int nb_cd_matches = 0;
722 #if !CONFIG_SMALL
723 AVBPrint bp;
724 #endif
725
726 /* We still accept functions marked with SLOW, even if the CPU is
727 * marked with the same flag, but we give them lower priority. */
728 14168 const int cpu_flags = av_get_cpu_flags();
729
730 /* Flags the transform wants */
731 14168 uint64_t req_flags = flags;
732
733 /* Flags the codelet may require to be present */
734 14168 uint64_t inv_req_mask = AV_TX_FULL_IMDCT |
735 AV_TX_REAL_TO_REAL |
736 AV_TX_REAL_TO_IMAGINARY |
737 FF_TX_PRESHUFFLE |
738 FF_TX_ASM_CALL;
739
740 /* Unaligned codelets are compatible with the aligned flag */
741
1/2
✓ Branch 0 taken 14168 times.
✗ Branch 1 not taken.
 14168 if (req_flags & FF_TX_ALIGNED)
742 14168 req_flags |= AV_TX_UNALIGNED;
743
744 /* If either flag is set, both are okay, so don't check for an exact match */
745
3/4
✓ Branch 0 taken 7273 times.
✓ Branch 1 taken 6895 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 7273 times.
 14168 if ((req_flags & AV_TX_INPLACE) && (req_flags & FF_TX_OUT_OF_PLACE))
746 ✗ req_flags &= ~(AV_TX_INPLACE | FF_TX_OUT_OF_PLACE);
747
2/4
✓ Branch 0 taken 14168 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 14168 times.
✗ Branch 3 not taken.
 14168 if ((req_flags & FF_TX_ALIGNED) && (req_flags & AV_TX_UNALIGNED))
748 14168 req_flags &= ~(FF_TX_ALIGNED | AV_TX_UNALIGNED);
749
750 /* Loop through all codelets in all codelet lists to find matches
751 * to the requirements */
752
2/2
✓ Branch 0 taken 70840 times.
✓ Branch 1 taken 14168 times.
 85008 while (codelet_list_idx--) {
753 70840 const FFTXCodelet * const * list = codelet_list[codelet_list_idx];
754 70840 const FFTXCodelet *cd = NULL;
755
756
2/2
✓ Branch 0 taken 3060288 times.
✓ Branch 1 taken 70840 times.
 3131128 while ((cd = *list++)) {
757 /* Check if the type matches */
758
4/4
✓ Branch 0 taken 3046120 times.
✓ Branch 1 taken 14168 times.
✓ Branch 2 taken 2404032 times.
✓ Branch 3 taken 642088 times.
 3060288 if (cd->type != TX_TYPE_ANY && type != cd->type)
759 2404032 continue;
760
761 /* Check direction for non-orthogonal codelets */
762
4/4
✓ Branch 0 taken 82731 times.
✓ Branch 1 taken 573525 times.
✓ Branch 2 taken 9331 times.
✓ Branch 3 taken 73400 times.
 656256 if (((cd->flags & FF_TX_FORWARD_ONLY) && inv) ||
763
4/4
✓ Branch 0 taken 70864 times.
✓ Branch 1 taken 511992 times.
✓ Branch 2 taken 64915 times.
✓ Branch 3 taken 5949 times.
 582856 ((cd->flags & (FF_TX_INVERSE_ONLY | AV_TX_FULL_IMDCT)) && !inv) ||
764
3/4
✓ Branch 0 taken 9331 times.
✓ Branch 1 taken 567576 times.
✓ Branch 2 taken 9331 times.
✗ Branch 3 not taken.
 576907 ((cd->flags & (FF_TX_FORWARD_ONLY | AV_TX_REAL_TO_REAL)) && inv) ||
765
3/4
✓ Branch 0 taken 9331 times.
✓ Branch 1 taken 567576 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 9331 times.
 576907 ((cd->flags & (FF_TX_FORWARD_ONLY | AV_TX_REAL_TO_IMAGINARY)) && inv))
766 79349 continue;
767
768 /* Check if the requested flags match from both sides */
769
2/2
✓ Branch 0 taken 353664 times.
✓ Branch 1 taken 223243 times.
 576907 if (((req_flags & cd->flags) != (req_flags)) ||
770
2/2
✓ Branch 0 taken 80081 times.
✓ Branch 1 taken 273583 times.
 353664 ((inv_req_mask & cd->flags) != (req_flags & inv_req_mask)))
771 303324 continue;
772
773 /* Check if length is supported */
774
6/6
✓ Branch 0 taken 186255 times.
✓ Branch 1 taken 87328 times.
✓ Branch 2 taken 127618 times.
✓ Branch 3 taken 58637 times.
✓ Branch 4 taken 105447 times.
✓ Branch 5 taken 22171 times.
 273583 if ((len < cd->min_len) || (cd->max_len != -1 && (len > cd->max_len)))
775 192775 continue;
776
777 /* Check if the CPU supports the required ISA */
778
2/2
✓ Branch 0 taken 24900 times.
✓ Branch 1 taken 55908 times.
 80808 if (cd->cpu_flags != FF_TX_CPU_FLAGS_ALL &&
779
2/2
✓ Branch 0 taken 17143 times.
✓ Branch 1 taken 7757 times.
 24900 !(cpu_flags & (cd->cpu_flags & ~cpu_slow_mask)))
780 17143 continue;
781
782 /* Check for factors */
783
2/2
✓ Branch 1 taken 26947 times.
✓ Branch 2 taken 36718 times.
 63665 if (!check_cd_factors(cd, len))
784 26947 continue;
785
786 /* Realloc array and append */
787 36718 cd_tmp = av_fast_realloc(cd_matches, &cd_matches_size,
788 36718 sizeof(*cd_tmp) * (nb_cd_matches + 1));
789
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36718 times.
 36718 if (!cd_tmp) {
790 ✗ av_free(cd_matches);
791 ✗ return AVERROR(ENOMEM);
792 }
793
794 36718 cd_matches = cd_tmp;
795 36718 cd_matches[nb_cd_matches].cd = cd;
796 36718 cd_matches[nb_cd_matches].prio = get_codelet_prio(cd, cpu_flags, len);
797 36718 nb_cd_matches++;
798 }
799 }
800
801 #if !CONFIG_SMALL
802 /* Print debugging info */
803 14168 av_bprint_init(&bp, 0, AV_BPRINT_SIZE_AUTOMATIC);
804
2/2
✓ Branch 0 taken 12396 times.
✓ Branch 1 taken 1772 times.
 14168 av_bprintf(&bp, "For transform of length %i, %s, ", len,
805 inv ? "inverse" : "forward");
806 14168 print_type(&bp, type);
807 14168 av_bprintf(&bp, ", ");
808 14168 print_flags(&bp, flags);
809
2/2
✓ Branch 0 taken 13954 times.
✓ Branch 1 taken 214 times.
 14168 av_bprintf(&bp, ", found %i matches%s", nb_cd_matches,
810 nb_cd_matches ? ":" : ".");
811 #endif
812
813 /* No matches found */
814
2/2
✓ Branch 0 taken 214 times.
✓ Branch 1 taken 13954 times.
 14168 if (!nb_cd_matches)
815 214 return AVERROR(ENOSYS);
816
817 /* Sort the list */
818
44/44
✓ Branch 0 taken 6479 times.
✓ Branch 1 taken 9636 times.
✓ Branch 3 taken 1675 times.
✓ Branch 4 taken 4804 times.
✓ Branch 6 taken 448 times.
✓ Branch 7 taken 1227 times.
✓ Branch 9 taken 2109 times.
✓ Branch 10 taken 2695 times.
✓ Branch 12 taken 2461 times.
✓ Branch 13 taken 4018 times.
✓ Branch 14 taken 3886 times.
✓ Branch 15 taken 2593 times.
✓ Branch 16 taken 4838 times.
✓ Branch 17 taken 1929 times.
✓ Branch 19 taken 3415 times.
✓ Branch 20 taken 1423 times.
✓ Branch 21 taken 1471 times.
✓ Branch 22 taken 2499 times.
✓ Branch 24 taken 618 times.
✓ Branch 25 taken 853 times.
✓ Branch 26 taken 2499 times.
✓ Branch 27 taken 853 times.
✓ Branch 28 taken 3352 times.
✓ Branch 29 taken 2593 times.
✓ Branch 30 taken 1082 times.
✓ Branch 31 taken 1511 times.
✓ Branch 32 taken 375 times.
✓ Branch 33 taken 707 times.
✓ Branch 34 taken 316 times.
✓ Branch 35 taken 59 times.
✓ Branch 36 taken 1958 times.
✓ Branch 37 taken 170 times.
✓ Branch 39 taken 1105 times.
✓ Branch 40 taken 853 times.
✓ Branch 41 taken 170 times.
✓ Branch 42 taken 853 times.
✓ Branch 43 taken 1779 times.
✓ Branch 44 taken 644 times.
✓ Branch 46 taken 6023 times.
✓ Branch 47 taken 3613 times.
✓ Branch 48 taken 16115 times.
✓ Branch 49 taken 2685 times.
✓ Branch 50 taken 16377 times.
✓ Branch 51 taken 13954 times.
 43837 AV_QSORT(cd_matches, nb_cd_matches, TXCodeletMatch, cmp_matches);
819
820 #if !CONFIG_SMALL
821 13954 av_log(NULL, AV_LOG_TRACE, "%s\n", bp.str);
822
823
2/2
✓ Branch 0 taken 36718 times.
✓ Branch 1 taken 13954 times.
 50672 for (int i = 0; i < nb_cd_matches; i++) {
824 36718 av_log(NULL, AV_LOG_TRACE, " %i: ", i + 1);
825 36718 print_cd_info(cd_matches[i].cd, cd_matches[i].prio, 0, 1, AV_LOG_TRACE);
826 }
827 #endif
828
829
2/2
✓ Branch 0 taken 13857 times.
✓ Branch 1 taken 97 times.
 13954 if (!s->sub) {
830 13857 s->sub = sub = av_mallocz(TX_MAX_SUB*sizeof(*sub));
831
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 13857 times.
 13857 if (!sub) {
832 ✗ ret = AVERROR(ENOMEM);
833 ✗ goto end;
834 }
835 }
836
837 /* Attempt to initialize each */
838
1/2
✓ Branch 0 taken 18811 times.
✗ Branch 1 not taken.
 18811 for (int i = 0; i < nb_cd_matches; i++) {
839 18811 const FFTXCodelet *cd = cd_matches[i].cd;
840 18811 AVTXContext *sctx = &s->sub[s->nb_sub];
841
842 18811 sctx->len = len;
843 18811 sctx->inv = inv;
844 18811 sctx->type = type;
845 18811 sctx->flags = cd->flags | flags;
846 18811 sctx->cd_self = cd;
847
848 18811 s->fn[s->nb_sub] = cd->function;
849 18811 s->cd[s->nb_sub] = cd;
850
851 18811 ret = 0;
852
2/2
✓ Branch 0 taken 18792 times.
✓ Branch 1 taken 19 times.
 18811 if (cd->init)
853 18792 ret = cd->init(sctx, cd, flags, opts, len, inv, scale);
854
855
2/2
✓ Branch 0 taken 13954 times.
✓ Branch 1 taken 4857 times.
 18811 if (ret >= 0) {
856
3/4
✓ Branch 0 taken 7164 times.
✓ Branch 1 taken 6790 times.
✓ Branch 2 taken 7164 times.
✗ Branch 3 not taken.
 13954 if (opts && opts->map_dir != FF_TX_MAP_NONE &&
857
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 7142 times.
 7164 sctx->map_dir == FF_TX_MAP_NONE) {
858 /* If a specific map direction was requested, and it doesn't
859 * exist, create one.*/
860 22 sctx->map = av_malloc(len*sizeof(*sctx->map));
861
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (!sctx->map) {
862 ✗ ret = AVERROR(ENOMEM);
863 ✗ goto end;
864 }
865
866
2/2
✓ Branch 0 taken 118 times.
✓ Branch 1 taken 22 times.
 140 for (int i = 0; i < len; i++)
867 118 sctx->map[i] = i;
868
4/4
✓ Branch 0 taken 7142 times.
✓ Branch 1 taken 6790 times.
✓ Branch 2 taken 16 times.
✓ Branch 3 taken 7126 times.
 13932 } else if (opts && (opts->map_dir != sctx->map_dir)) {
869 16 int *tmp = av_malloc(len*sizeof(*sctx->map));
870
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
 16 if (!tmp) {
871 ✗ ret = AVERROR(ENOMEM);
872 ✗ goto end;
873 }
874
875 16 memcpy(tmp, sctx->map, len*sizeof(*sctx->map));
876
877
2/2
✓ Branch 0 taken 112 times.
✓ Branch 1 taken 16 times.
 128 for (int i = 0; i < len; i++)
878 112 sctx->map[tmp[i]] = i;
879
880 16 av_free(tmp);
881 }
882
883 13954 s->nb_sub++;
884 13954 goto end;
885 }
886
887 4857 s->fn[s->nb_sub] = NULL;
888 4857 s->cd[s->nb_sub] = NULL;
889
890 4857 reset_ctx(sctx, 0);
891
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4857 times.
 4857 if (ret == AVERROR(ENOMEM))
892 ✗ break;
893 }
894
895 ✗ if (!s->nb_sub)
896 ✗ av_freep(&s->sub);
897
898 ✗ end:
899 13954 av_free(cd_matches);
900 13954 return ret;
901 }
902
903 6603 av_cold int av_tx_init(AVTXContext **ctx, av_tx_fn *tx, enum AVTXType type,
904 int inv, int len, const void *scale, uint64_t flags)
905 {
906 int ret;
907 6603 AVTXContext tmp = { 0 };
908 6603 const double default_scale_d = 1.0;
909 6603 const float default_scale_f = 1.0f;
910
911
4/8
✓ Branch 0 taken 6603 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6603 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 6603 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 6603 times.
 6603 if (!len || type >= AV_TX_NB || !ctx || !tx)
912 ✗ return AVERROR(EINVAL);
913
914
1/2
✓ Branch 0 taken 6603 times.
✗ Branch 1 not taken.
 6603 if (!(flags & AV_TX_UNALIGNED))
915 6603 flags |= FF_TX_ALIGNED;
916
1/2
✓ Branch 0 taken 6603 times.
✗ Branch 1 not taken.
 6603 if (!(flags & AV_TX_INPLACE))
917 6603 flags |= FF_TX_OUT_OF_PLACE;
918
919
5/8
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 6591 times.
✓ Branch 2 taken 12 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 12 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 12 times.
✗ Branch 7 not taken.
 6603 if (!scale && ((type == AV_TX_DOUBLE_MDCT) || (type == AV_TX_DOUBLE_DCT) ||
920
2/4
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
 12 (type == AV_TX_DOUBLE_DCT_I) || (type == AV_TX_DOUBLE_DST_I) ||
921 (type == AV_TX_DOUBLE_RDFT)))
922 ✗ scale = &default_scale_d;
923
3/8
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 6591 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 12 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 6603 else if (!scale && !TYPE_IS(FFT, type))
924 ✗ scale = &default_scale_f;
925
926 6603 ret = ff_tx_init_subtx(&tmp, type, flags, NULL, len, inv, scale);
927
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6603 times.
 6603 if (ret < 0)
928 ✗ return ret;
929
930 6603 *ctx = &tmp.sub[0];
931 6603 *tx = tmp.fn[0];
932
933 #if !CONFIG_SMALL
934 6603 av_log(NULL, AV_LOG_DEBUG, "Transform tree:\n");
935 6603 print_tx_structure(*ctx, 0);
936 #endif
937
938 6603 return ret;
939 }
940