FFmpeg coverage

Directory: ../../../ffmpeg/
File: src/libswscale/x86/ops.c
Date: 2026-07-19 01:11:25
Exec Total Coverage
Lines: 230 268 85.8%
Functions: 18 18 100.0%
Branches: 176 332 53.0%
Line Branch Exec Source
1 /**
2 * Copyright (C) 2025-2026 Niklas Haas
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include <float.h>
22
23 #include "libavutil/avassert.h"
24 #include "libavutil/mem.h"
25 #include "libavutil/x86/cpu.h"
26
27 #include "../ops_chain.h"
28 #include "../uops.h"
29 #include "../uops_macros.h"
30
31 21853 static int setup_rw_packed(const SwsImplParams *params, SwsImplResult *out)
32 {
33 21853 const SwsUOp *uop = params->uop;
34
35 /* 3-component packed reads/writes process one extra garbage word */
36
2/2
✓ Branch 0 taken 5646 times.
✓ Branch 1 taken 16207 times.
 21853 if (uop->mask == SWS_COMP_ELEMS(3)) {
37
2/3
✓ Branch 0 taken 3172 times.
✓ Branch 1 taken 2474 times.
✗ Branch 2 not taken.
 5646 switch (uop->uop) {
38 3172 case SWS_UOP_READ_PACKED: out->over_read[0] = sizeof(uint32_t); break;
39 2474 case SWS_UOP_WRITE_PACKED: out->over_write[0] = sizeof(uint32_t); break;
40 }
41 }
42
43 21853 return 0;
44 }
45
46 18932 static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out)
47 {
48 18932 const SwsFilterWeights *filter = params->uop->data.kernel;
49 static_assert(sizeof(out->priv.ptr) <= sizeof(int32_t[2]),
50 ">8 byte pointers not supported");
51
52 /* Pre-convert weights to float */
53 18932 float *weights = av_calloc(filter->num_weights, sizeof(float));
54
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 18932 times.
 18932 if (!weights)
55 ✗ return AVERROR(ENOMEM);
56
57
2/2
✓ Branch 0 taken 6211520 times.
✓ Branch 1 taken 18932 times.
 6230452 for (int i = 0; i < filter->num_weights; i++)
58 6211520 weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE;
59
60 18932 out->priv.ptr = weights;
61 18932 out->priv.uptr[1] = filter->filter_size;
62 18932 out->free = ff_op_priv_free;
63 18932 return 0;
64 }
65
66 19294 static int hscale_sizeof_weight(const SwsUOp *uop)
67 {
68
3/4
✓ Branch 0 taken 5648 times.
✓ Branch 1 taken 10591 times.
✓ Branch 2 taken 3055 times.
✗ Branch 3 not taken.
 19294 switch (uop->type) {
69 5648 case SWS_PIXEL_U8: return sizeof(int16_t);
70 10591 case SWS_PIXEL_U16: return sizeof(int16_t);
71 3055 case SWS_PIXEL_F32: return sizeof(float);
72 ✗ default: return 0;
73 }
74 }
75
76 1690 static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out)
77 {
78 1690 const SwsUOp *uop = params->uop;
79 1690 const SwsFilterWeights *filter = uop->data.kernel;
80
81 /**
82 * `vpgatherdd` gathers 32 bits at a time; so if we're filtering a smaller
83 * size, we need to gather 2/4 taps simultaneously and unroll the inner
84 * loop over several packed samples.
85 */
86 1690 const int pixel_size = ff_sws_pixel_type_size(uop->type);
87 1690 const int taps_align = sizeof(int32_t) / pixel_size;
88 1690 const int filter_size = filter->filter_size;
89 1690 const int block_size = params->table->block_size;
90 1690 const size_t aligned_size = FFALIGN(filter_size, taps_align);
91 1690 const size_t line_size = FFALIGN(filter->dst_size, block_size);
92 av_assert1(FFALIGN(line_size, taps_align) == line_size);
93
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1690 times.
 1690 if (aligned_size > INT_MAX)
94 ✗ return AVERROR(EINVAL);
95
96 union {
97 void *ptr;
98 int16_t *i16;
99 float *f32;
100 } weights;
101
102 1690 const int sizeof_weight = hscale_sizeof_weight(uop);
103 1690 weights.ptr = av_calloc(line_size, sizeof_weight * aligned_size);
104
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1690 times.
 1690 if (!weights.ptr)
105 ✗ return AVERROR(ENOMEM);
106
107 /**
108 * Transpose filter weights to group (aligned) taps by block
109 */
110 1690 const int mmsize = block_size * 2;
111 1690 const int gather_size = mmsize / sizeof(int32_t); /* pixels per vpgatherdd */
112
2/2
✓ Branch 0 taken 10046 times.
✓ Branch 1 taken 1690 times.
 11736 for (size_t x = 0; x < line_size; x += block_size) {
113 10046 const int elems = FFMIN(block_size, filter->dst_size - x);
114
2/2
✓ Branch 0 taken 28114 times.
✓ Branch 1 taken 10046 times.
 38160 for (int j = 0; j < filter_size; j++) {
115 28114 const int jb = j & ~(taps_align - 1);
116 28114 const int ji = j - jb;
117 28114 const size_t idx_base = x * aligned_size + jb * block_size + ji;
118
2/2
✓ Branch 0 taken 449824 times.
✓ Branch 1 taken 28114 times.
 477938 for (int i = 0; i < elems; i++) {
119 449824 const int w = filter->weights[(x + i) * filter_size + j];
120 449824 size_t idx = idx_base;
121
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 449824 times.
 449824 if (uop->type == SWS_PIXEL_U8) {
122 /* Interleave the pixels within each lane, i.e.:
123 * [a0 a1 a2 a3 | b0 b1 b2 b3 ] pixels 0-1, taps 0-3 (lane 0)
124 * [e0 e1 e2 e3 | f0 f1 f2 f3 ] pixels 4-5, taps 0-3 (lane 1)
125 * [c0 c1 c2 c3 | d0 d1 d2 d3 ] pixels 2-3, taps 0-3 (lane 0)
126 * [g0 g1 g2 g3 | h0 h1 h2 h3 ] pixels 6-7, taps 0-3 (lane 1)
127 * [i0 i1 i2 i3 | j0 j1 j2 j3 ] pixels 8-9, taps 0-3 (lane 0)
128 * ...
129 * [o0 o1 o2 o3 | p0 p1 p2 p3 ] pixels 14-15, taps 0-3 (lane 1)
130 * (repeat for taps 4-7, etc.)
131 */
132 ✗ const int gather_base = i & ~(gather_size - 1);
133 ✗ const int gather_pos = i - gather_base;
134 ✗ const int lane_idx = gather_pos >> 2;
135 ✗ const int pos_in_lane = gather_pos & 3;
136 ✗ idx += gather_base * 4 /* which gather (m0 or m1) */
137 ✗ + (pos_in_lane >> 1) * (mmsize / 2) /* lo/hi unpack */
138 ✗ + lane_idx * 8 /* 8 ints per lane */
139 ✗ + (pos_in_lane & 1) * 4; /* 4 taps per pair */
140 } else {
141 449824 idx += i * taps_align;
142 }
143
144
1/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 449824 times.
✗ Branch 3 not taken.
 449824 switch (uop->type) {
145 ✗ case SWS_PIXEL_U8: weights.i16[idx] = w; break;
146 ✗ case SWS_PIXEL_U16: weights.i16[idx] = w; break;
147 449824 case SWS_PIXEL_F32: weights.f32[idx] = w; break;
148 }
149 }
150 }
151 }
152
153 1690 out->priv.ptr = weights.ptr;
154 1690 out->priv.uptr[1] = aligned_size;
155 1690 out->free = ff_op_priv_free;
156
157
2/2
✓ Branch 0 taken 6760 times.
✓ Branch 1 taken 1690 times.
 8450 for (int i = 0; i < 4; i++) {
158
2/2
✓ Branch 0 taken 4742 times.
✓ Branch 1 taken 2018 times.
 6760 if (uop->mask & SWS_COMP(i))
159 4742 out->over_read[i] = (aligned_size - filter_size) * pixel_size;
160 }
161 1690 return 0;
162 }
163
164 19294 static bool check_filter_h_4x4(const SwsImplParams *params)
165 {
166 19294 SwsContext *ctx = params->ctx;
167 19294 const SwsUOp *uop = params->uop;
168
4/4
✓ Branch 0 taken 4399 times.
✓ Branch 1 taken 14895 times.
✓ Branch 2 taken 1690 times.
✓ Branch 3 taken 2709 times.
 19294 if ((ctx->flags & SWS_BITEXACT) && uop->type == SWS_PIXEL_F32)
169 1690 return false; /* different accumulation order due to 4x4 transpose */
170
171 17604 const int cpu_flags = av_get_cpu_flags();
172
1/2
✓ Branch 0 taken 17604 times.
✗ Branch 1 not taken.
 17604 if (cpu_flags & AV_CPU_FLAG_SLOW_GATHER)
173 17604 return true; /* always prefer over gathers if gathers are slow */
174
175 /**
176 * Otherwise, prefer it above a certain filter size. Empirically, this
177 * kernel seems to be faster whenever the reference/gather kernel crosses
178 * a breakpoint for the number of gathers needed, but this filter doesn't.
179 *
180 * Tested on a Lunar Lake (Intel Core Ultra 7 258V) system.
181 */
182 ✗ const SwsFilterWeights *filter = uop->data.kernel;
183 ✗ return uop->type == SWS_PIXEL_U8 && filter->filter_size > 12 ||
184 ✗ uop->type == SWS_PIXEL_U16 && filter->filter_size > 4 ||
185 ✗ uop->type == SWS_PIXEL_F32 && filter->filter_size > 1;
186 }
187
188 17604 static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out)
189 {
190 17604 const SwsUOp *uop = params->uop;
191 17604 const SwsFilterWeights *filter = uop->data.kernel;
192 17604 const int pixel_size = ff_sws_pixel_type_size(uop->type);
193 17604 const int sizeof_weights = hscale_sizeof_weight(uop);
194 17604 const int block_size = params->table->block_size;
195 17604 const int taps_align = 16 / sizeof_weights; /* taps per iteration (XMM) */
196 17604 const int pixels_align = 4; /* pixels per iteration */
197 17604 const int filter_size = filter->filter_size;
198 17604 const size_t aligned_size = FFALIGN(filter_size, taps_align);
199 17604 const int line_size = FFALIGN(filter->dst_size, block_size);
200 av_assert1(FFALIGN(line_size, pixels_align) == line_size);
201
202 union {
203 void *ptr;
204 int16_t *i16;
205 float *f32;
206 } weights;
207
208 17604 weights.ptr = av_calloc(line_size, aligned_size * sizeof_weights);
209
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17604 times.
 17604 if (!weights.ptr)
210 ✗ return AVERROR(ENOMEM);
211
212 /**
213 * Desired memory layout: [w][taps][pixels_align][taps_align]
214 *
215 * Example with taps_align=8, pixels_align=4:
216 * [a0, a1, ... a7] weights for pixel 0, taps 0..7
217 * [b0, b1, ... b7] weights for pixel 1, taps 0..7
218 * [c0, c1, ... c7] weights for pixel 2, taps 0..7
219 * [d0, d1, ... d7] weights for pixel 3, taps 0..7
220 * [a8, a9, ... a15] weights for pixel 0, taps 8..15
221 * ...
222 * repeat for all taps, then move on to pixels 4..7, etc.
223 */
224
2/2
✓ Branch 0 taken 1691584 times.
✓ Branch 1 taken 17604 times.
 1709188 for (int x = 0; x < filter->dst_size; x++) {
225
2/2
✓ Branch 0 taken 5917760 times.
✓ Branch 1 taken 1691584 times.
 7609344 for (int j = 0; j < filter_size; j++) {
226 5917760 const int xb = x & ~(pixels_align - 1);
227 5917760 const int jb = j & ~(taps_align - 1);
228 5917760 const int xi = x - xb, ji = j - jb;
229 5917760 const int w = filter->weights[x * filter_size + j];
230 5917760 const int idx = xb * aligned_size + jb * pixels_align + xi * taps_align + ji;
231
232
3/4
✓ Branch 0 taken 1899584 times.
✓ Branch 1 taken 3577920 times.
✓ Branch 2 taken 440256 times.
✗ Branch 3 not taken.
 5917760 switch (uop->type) {
233 1899584 case SWS_PIXEL_U8: weights.i16[idx] = w; break;
234 3577920 case SWS_PIXEL_U16: weights.i16[idx] = w; break;
235 440256 case SWS_PIXEL_F32: weights.f32[idx] = w; break;
236 }
237 }
238 }
239
240 17604 out->priv.ptr = weights.ptr;
241 17604 out->priv.uptr[1] = aligned_size * sizeof_weights;
242 17604 out->free = ff_op_priv_free;
243
244
2/2
✓ Branch 0 taken 70416 times.
✓ Branch 1 taken 17604 times.
 88020 for (int i = 0; i < 4; i++) {
245
2/2
✓ Branch 0 taken 47861 times.
✓ Branch 1 taken 22555 times.
 70416 if (uop->mask & SWS_COMP(i))
246 47861 out->over_read[i] = (aligned_size - filter_size) * pixel_size;
247 }
248 17604 return 0;
249 }
250
251 12051 static int setup_scale(const SwsImplParams *params, SwsImplResult *out)
252 {
253 12051 const SwsUOp *uop = params->uop;
254
4/5
✓ Branch 0 taken 67 times.
✓ Branch 1 taken 177 times.
✓ Branch 2 taken 22 times.
✓ Branch 3 taken 11785 times.
✗ Branch 4 not taken.
 12051 switch (uop->type) {
255 67 case SWS_PIXEL_U8: out->priv.u16[0] = uop->data.scalar.u8; break; /* for pmullw */
256 177 case SWS_PIXEL_U16: out->priv.u16[0] = uop->data.scalar.u16; break;
257 22 case SWS_PIXEL_U32: out->priv.u32[0] = uop->data.scalar.u32; break;
258 11785 case SWS_PIXEL_F32: out->priv.f32[0] = uop->data.scalar.f32; break;
259 ✗ default: return AVERROR(EINVAL);
260 }
261
262 12051 return 0;
263 }
264
265 6585 static int setup_clear(const SwsImplParams *params, SwsImplResult *out)
266 {
267 6585 const SwsUOp *uop = params->uop;
268
2/2
✓ Branch 0 taken 26340 times.
✓ Branch 1 taken 6585 times.
 32925 for (int i = 0; i < 4; i++)
269 26340 out->priv.u32[i] = uop->data.vec4[i].u32;
270 6585 return 0;
271 }
272
273 24592 static int setup_dither(const SwsImplParams *params, SwsImplResult *out)
274 {
275 24592 out->priv.ptr = av_refstruct_ref(params->uop->data.ptr);
276 24592 out->free = ff_op_priv_unref;
277 24592 return 0;
278 }
279
280 28080 static int setup_linear(const SwsImplParams *params, SwsImplResult *out)
281 {
282 28080 const SwsUOp *uop = params->uop;
283 28080 out->priv.ptr = av_memdup(uop->data.mat4, sizeof(uop->data.mat4));
284 28080 out->free = ff_op_priv_free;
285
1/2
✓ Branch 0 taken 28080 times.
✗ Branch 1 not taken.
 28080 return out->priv.ptr ? 0 : AVERROR(ENOMEM);
286 }
287
288 408828 static bool uop_is_type_invariant(const SwsUOpType uop)
289 {
290
2/2
✓ Branch 0 taken 95751 times.
✓ Branch 1 taken 313077 times.
 408828 switch (uop) {
291 95751 case SWS_UOP_READ_PLANAR:
292 case SWS_UOP_WRITE_PLANAR:
293 case SWS_UOP_CLEAR:
294 95751 return true;
295 313077 default:
296 313077 return false;
297 }
298 }
299
300 #define REF_ENTRY(EXT, NAME, ...) &uop_##NAME##EXT,
301 #define DECL_ENTRY(EXT, CHECK, SETUP, NAME, ...) \
302 void ff_##NAME##EXT(void); \
303 static const SwsUOpEntry uop_##NAME##EXT = { \
304 .func = (SwsFuncPtr) ff_##NAME##EXT, \
305 .check = CHECK, \
306 .setup = SETUP, \
307 __VA_ARGS__, \
308 };
309
310 /* Define all UOPs except conversion ops and type-invariant ops */
311 #define DECL_OPS_COMMON(EXT, TYPE) \
312 SWS_FOR_STRUCT(TYPE, READ_PACKED, DECL_ENTRY, EXT, NULL, setup_rw_packed) \
313 SWS_FOR_STRUCT(TYPE, READ_NIBBLE, DECL_ENTRY, EXT, NULL, NULL) \
314 SWS_FOR_STRUCT(TYPE, READ_BIT, DECL_ENTRY, EXT, NULL, NULL) \
315 SWS_FOR_STRUCT(TYPE, READ_PALETTE, DECL_ENTRY, EXT, NULL, NULL) \
316 SWS_FOR_STRUCT(TYPE, WRITE_PACKED, DECL_ENTRY, EXT, NULL, setup_rw_packed) \
317 SWS_FOR_STRUCT(TYPE, WRITE_NIBBLE, DECL_ENTRY, EXT, NULL, NULL) \
318 SWS_FOR_STRUCT(TYPE, WRITE_BIT, DECL_ENTRY, EXT, NULL, NULL) \
319 SWS_FOR_STRUCT(TYPE, SWAP_BYTES, DECL_ENTRY, EXT, NULL, NULL) \
320 SWS_FOR_STRUCT(TYPE, EXPAND_BIT, DECL_ENTRY, EXT, NULL, NULL) \
321 SWS_FOR_STRUCT(TYPE, PERMUTE, DECL_ENTRY, EXT, NULL, NULL) \
322 SWS_FOR_STRUCT(TYPE, COPY, DECL_ENTRY, EXT, NULL, NULL) \
323 SWS_FOR_STRUCT(TYPE, SCALE, DECL_ENTRY, EXT, NULL, setup_scale) \
324 SWS_FOR_STRUCT(TYPE, ADD, DECL_ENTRY, EXT, NULL, ff_sws_setup_vec4) \
325 SWS_FOR_STRUCT(TYPE, MIN, DECL_ENTRY, EXT, NULL, ff_sws_setup_vec4) \
326 SWS_FOR_STRUCT(TYPE, MAX, DECL_ENTRY, EXT, NULL, ff_sws_setup_vec4) \
327 SWS_FOR_STRUCT(TYPE, UNPACK, DECL_ENTRY, EXT, NULL, NULL) \
328 SWS_FOR_STRUCT(TYPE, PACK, DECL_ENTRY, EXT, NULL, NULL) \
329 SWS_FOR_STRUCT(TYPE, LSHIFT, DECL_ENTRY, EXT, NULL, NULL) \
330 SWS_FOR_STRUCT(TYPE, RSHIFT, DECL_ENTRY, EXT, NULL, NULL) \
331 SWS_FOR_STRUCT(TYPE, LINEAR, DECL_ENTRY, EXT, NULL, setup_linear) \
332 SWS_FOR_STRUCT(TYPE, LINEAR_FMA, DECL_ENTRY, EXT, NULL, setup_linear) \
333 SWS_FOR_STRUCT(TYPE, DITHER, DECL_ENTRY, EXT, NULL, setup_dither) \
334 /* end of macro */
335
336 #define REF_OPS_COMMON(EXT, TYPE) \
337 SWS_FOR(TYPE, READ_PACKED, REF_ENTRY, EXT) \
338 SWS_FOR(TYPE, READ_NIBBLE, REF_ENTRY, EXT) \
339 SWS_FOR(TYPE, READ_BIT, REF_ENTRY, EXT) \
340 SWS_FOR(TYPE, READ_PALETTE, REF_ENTRY, EXT) \
341 SWS_FOR(TYPE, WRITE_PACKED, REF_ENTRY, EXT) \
342 SWS_FOR(TYPE, WRITE_NIBBLE, REF_ENTRY, EXT) \
343 SWS_FOR(TYPE, WRITE_BIT, REF_ENTRY, EXT) \
344 SWS_FOR(TYPE, SWAP_BYTES, REF_ENTRY, EXT) \
345 SWS_FOR(TYPE, EXPAND_BIT, REF_ENTRY, EXT) \
346 SWS_FOR(TYPE, PERMUTE, REF_ENTRY, EXT) \
347 SWS_FOR(TYPE, COPY, REF_ENTRY, EXT) \
348 SWS_FOR(TYPE, SCALE, REF_ENTRY, EXT) \
349 SWS_FOR(TYPE, ADD, REF_ENTRY, EXT) \
350 SWS_FOR(TYPE, MIN, REF_ENTRY, EXT) \
351 SWS_FOR(TYPE, MAX, REF_ENTRY, EXT) \
352 SWS_FOR(TYPE, UNPACK, REF_ENTRY, EXT) \
353 SWS_FOR(TYPE, PACK, REF_ENTRY, EXT) \
354 SWS_FOR(TYPE, LSHIFT, REF_ENTRY, EXT) \
355 SWS_FOR(TYPE, RSHIFT, REF_ENTRY, EXT) \
356 SWS_FOR(TYPE, LINEAR, REF_ENTRY, EXT) \
357 SWS_FOR(TYPE, LINEAR_FMA, REF_ENTRY, EXT) \
358 SWS_FOR(TYPE, DITHER, REF_ENTRY, EXT) \
359 /* end of macro */
360
361 #define DECL_TABLE_U8(EXT, SIZE, FLAG) \
362 DECL_OPS_COMMON(EXT, U8) \
363 SWS_FOR_STRUCT(U8, READ_PLANAR, DECL_ENTRY, EXT, NULL, NULL) \
364 SWS_FOR_STRUCT(U8, WRITE_PLANAR, DECL_ENTRY, EXT, NULL, NULL) \
365 SWS_FOR_STRUCT(U8, CLEAR, DECL_ENTRY, EXT, NULL, setup_clear) \
366 \
367 static const SwsUOpTable uops_u8##EXT = { \
368 .cpu_flags = AV_CPU_FLAG_##FLAG, \
369 .block_size = SIZE, \
370 .entries = { \
371 REF_OPS_COMMON(EXT, U8) \
372 SWS_FOR(U8, READ_PLANAR, REF_ENTRY, EXT) \
373 SWS_FOR(U8, WRITE_PLANAR, REF_ENTRY, EXT) \
374 SWS_FOR(U8, CLEAR, REF_ENTRY, EXT) \
375 NULL \
376 }, \
377 };
378
379 #define DECL_TABLE_U16(EXT, SIZE, FLAG) \
380 DECL_OPS_COMMON(EXT, U16) \
381 SWS_FOR_STRUCT(U8, TO_U16, DECL_ENTRY, EXT, NULL, NULL) \
382 SWS_FOR_STRUCT(U16, TO_U8, DECL_ENTRY, EXT, NULL, NULL) \
383 SWS_FOR_STRUCT(U8, EXPAND_PAIR, DECL_ENTRY, EXT, NULL, NULL) \
384 \
385 static const SwsUOpTable uops_u16##EXT = { \
386 .cpu_flags = AV_CPU_FLAG_##FLAG, \
387 .block_size = SIZE, \
388 .entries = { \
389 REF_OPS_COMMON(EXT, U16) \
390 SWS_FOR(U8, TO_U16, REF_ENTRY, EXT) \
391 SWS_FOR(U16, TO_U8, REF_ENTRY, EXT) \
392 SWS_FOR(U8, EXPAND_PAIR, REF_ENTRY, EXT) \
393 NULL \
394 }, \
395 };
396
397 #define DECL_TABLE_U32(EXT, SIZE, FLAG) \
398 DECL_OPS_COMMON(EXT, U32) \
399 SWS_FOR_STRUCT(U8, TO_U32, DECL_ENTRY, EXT, NULL, NULL) \
400 SWS_FOR_STRUCT(U32, TO_U8, DECL_ENTRY, EXT, NULL, NULL) \
401 SWS_FOR_STRUCT(U16, TO_U32, DECL_ENTRY, EXT, NULL, NULL) \
402 SWS_FOR_STRUCT(U32, TO_U16, DECL_ENTRY, EXT, NULL, NULL) \
403 SWS_FOR_STRUCT(U8, EXPAND_QUAD, DECL_ENTRY, EXT, NULL, NULL) \
404 \
405 static const SwsUOpTable uops_u32##EXT = { \
406 .cpu_flags = AV_CPU_FLAG_##FLAG, \
407 .block_size = SIZE, \
408 .entries = { \
409 REF_OPS_COMMON(EXT, U32) \
410 SWS_FOR(U8, TO_U32, REF_ENTRY, EXT) \
411 SWS_FOR(U32, TO_U8, REF_ENTRY, EXT) \
412 SWS_FOR(U16, TO_U32, REF_ENTRY, EXT) \
413 SWS_FOR(U32, TO_U16, REF_ENTRY, EXT) \
414 SWS_FOR(U8, EXPAND_QUAD, REF_ENTRY, EXT) \
415 NULL \
416 }, \
417 };
418
419 #define DECL_TABLE_F32(EXT, SIZE, FLAG) \
420 DECL_OPS_COMMON(EXT, F32) \
421 SWS_FOR_STRUCT(U8, TO_F32, DECL_ENTRY, EXT, NULL, NULL) \
422 SWS_FOR_STRUCT(F32, TO_U8, DECL_ENTRY, EXT, NULL, NULL) \
423 SWS_FOR_STRUCT(U16, TO_F32, DECL_ENTRY, EXT, NULL, NULL) \
424 SWS_FOR_STRUCT(F32, TO_U16, DECL_ENTRY, EXT, NULL, NULL) \
425 SWS_FOR_STRUCT(U8, READ_PLANAR_FH, DECL_ENTRY, EXT, NULL, setup_filter_h) \
426 SWS_FOR_STRUCT(U16, READ_PLANAR_FH, DECL_ENTRY, EXT, NULL, setup_filter_h) \
427 SWS_FOR_STRUCT(F32, READ_PLANAR_FH, DECL_ENTRY, EXT, NULL, setup_filter_h) \
428 SWS_FOR_STRUCT(U8, READ_PLANAR_FH, DECL_ENTRY, _4x4##EXT, \
429 check_filter_h_4x4, setup_filter_h_4x4) \
430 SWS_FOR_STRUCT(U16, READ_PLANAR_FH, DECL_ENTRY, _4x4##EXT, \
431 check_filter_h_4x4, setup_filter_h_4x4) \
432 SWS_FOR_STRUCT(F32, READ_PLANAR_FH, DECL_ENTRY, _4x4##EXT, \
433 check_filter_h_4x4, setup_filter_h_4x4) \
434 SWS_FOR_STRUCT(U8, READ_PLANAR_FV, DECL_ENTRY, EXT, NULL, setup_filter_v) \
435 SWS_FOR_STRUCT(U16, READ_PLANAR_FV, DECL_ENTRY, EXT, NULL, setup_filter_v) \
436 SWS_FOR_STRUCT(F32, READ_PLANAR_FV, DECL_ENTRY, EXT, NULL, setup_filter_v) \
437 SWS_FOR_STRUCT(U8, READ_PLANAR_FV_FMA, DECL_ENTRY, EXT, NULL, setup_filter_v) \
438 SWS_FOR_STRUCT(U16, READ_PLANAR_FV_FMA, DECL_ENTRY, EXT, NULL, setup_filter_v) \
439 SWS_FOR_STRUCT(F32, READ_PLANAR_FV_FMA, DECL_ENTRY, EXT, NULL, setup_filter_v) \
440 \
441 static const SwsUOpTable uops_f32##EXT = { \
442 .cpu_flags = AV_CPU_FLAG_##FLAG, \
443 .block_size = SIZE, \
444 .entries = { \
445 REF_OPS_COMMON(EXT, F32) \
446 SWS_FOR(U8, TO_F32, REF_ENTRY, EXT) \
447 SWS_FOR(F32, TO_U8, REF_ENTRY, EXT) \
448 SWS_FOR(U16, TO_F32, REF_ENTRY, EXT) \
449 SWS_FOR(F32, TO_U16, REF_ENTRY, EXT) \
450 SWS_FOR(U8, READ_PLANAR_FH, REF_ENTRY, _4x4##EXT) \
451 SWS_FOR(U16, READ_PLANAR_FH, REF_ENTRY, _4x4##EXT) \
452 SWS_FOR(F32, READ_PLANAR_FH, REF_ENTRY, _4x4##EXT) \
453 SWS_FOR(U8, READ_PLANAR_FH, REF_ENTRY, EXT) \
454 SWS_FOR(U16, READ_PLANAR_FH, REF_ENTRY, EXT) \
455 SWS_FOR(F32, READ_PLANAR_FH, REF_ENTRY, EXT) \
456 SWS_FOR(U8, READ_PLANAR_FV, REF_ENTRY, EXT) \
457 SWS_FOR(U16, READ_PLANAR_FV, REF_ENTRY, EXT) \
458 SWS_FOR(F32, READ_PLANAR_FV, REF_ENTRY, EXT) \
459 SWS_FOR(U8, READ_PLANAR_FV_FMA, REF_ENTRY, EXT) \
460 SWS_FOR(U16, READ_PLANAR_FV_FMA, REF_ENTRY, EXT) \
461 SWS_FOR(F32, READ_PLANAR_FV_FMA, REF_ENTRY, EXT) \
462 NULL \
463 }, \
464 };
465
466 DECL_TABLE_U8( _m1_sse4, 16, SSE4)
467 DECL_TABLE_U8( _m1_avx2, 32, AVX2)
468 DECL_TABLE_U8( _m2_sse4, 32, SSE4)
469 DECL_TABLE_U8( _m2_avx2, 64, AVX2)
470 DECL_TABLE_U16(_m1_avx2, 16, AVX2)
471 DECL_TABLE_U16(_m2_avx2, 32, AVX2)
472 DECL_TABLE_U32(_m2_avx2, 16, AVX2)
473 DECL_TABLE_F32(_m2_avx2, 16, AVX2)
474
475 static const SwsUOpTable *const tables[] = {
476 &uops_u8_m1_sse4,
477 &uops_u8_m1_avx2, /* order before _m2_sse4 */
478 &uops_u8_m2_sse4,
479 &uops_u8_m2_avx2,
480 &uops_u16_m1_avx2,
481 &uops_u16_m2_avx2,
482 &uops_u32_m2_avx2,
483 &uops_f32_m2_avx2,
484 };
485
486 SWS_DECL_FUNC(ff_sws_process1_x86);
487 SWS_DECL_FUNC(ff_sws_process2_x86);
488 SWS_DECL_FUNC(ff_sws_process3_x86);
489 SWS_DECL_FUNC(ff_sws_process4_x86);
490
491 /* Declare packed shuffle functions */
492 SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _sse4, NULL, NULL)
493 SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx2, NULL, NULL)
494 SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx512, NULL, NULL)
495 SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx512icl, NULL, NULL)
496
497 201694 static int get_mmsize(void)
498 {
499 201694 const int cpu_flags = av_get_cpu_flags();
500
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 201694 times.
 201694 if (EXTERNAL_AVX512(cpu_flags))
501 ✗ return 64;
502
2/2
✓ Branch 0 taken 193491 times.
✓ Branch 1 taken 8203 times.
 201694 else if (EXTERNAL_AVX2(cpu_flags))
503 193491 return 32;
504
2/2
✓ Branch 0 taken 3786 times.
✓ Branch 1 taken 4417 times.
 8203 else if (EXTERNAL_SSE4(cpu_flags))
505 3786 return 16;
506 else
507 4417 return AVERROR(ENOTSUP);
508 }
509
510 5130 static int movsize(const int bytes, const int mmsize)
511 {
512
2/2
✓ Branch 0 taken 4895 times.
✓ Branch 1 taken 235 times.
 10025 return bytes <= 4 ? 4 : /* movd */
513
2/2
✓ Branch 0 taken 4202 times.
✓ Branch 1 taken 693 times.
 9097 bytes <= 8 ? 8 : /* movq */
514
2/2
✓ Branch 0 taken 3086 times.
✓ Branch 1 taken 1116 times.
 7288 bytes <= 16 ? 16 : /* xmm movu */
515
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3086 times.
 3086 bytes <= 32 ? 32 : /* ymm movu */
516 mmsize; /* zmm movu */
517 }
518
519 2565 static int translate_shuffle(const SwsUOp *uop, int mmsize, SwsCompiledOp *out)
520 {
521 /* We can't shuffle across lanes, so restrict the vector size to XMM
522 * whenever the read/write size would be a subset of the full vector,
523 * unless we have access to AVX-512 ICL vpermb */
524 2565 const SwsShuffleUOp *par = &uop->par.shuffle;
525
2/2
✓ Branch 0 taken 1559 times.
✓ Branch 1 taken 1006 times.
 4124 const int lane_aligned = par->read_size == par->write_size &&
526
2/2
✓ Branch 0 taken 1543 times.
✓ Branch 1 taken 16 times.
 1559 16 % par->read_size == 0;
527
3/4
✓ Branch 0 taken 1022 times.
✓ Branch 1 taken 1543 times.
✓ Branch 3 taken 1022 times.
✗ Branch 4 not taken.
 2565 if (!lane_aligned && !EXTERNAL_AVX512ICL(av_get_cpu_flags()))
528 1022 mmsize = 16;
529
530 /* Generate the shuffle mask */
531
2/2
✓ Branch 0 taken 1022 times.
✓ Branch 1 taken 1543 times.
 2565 const int mask_size = lane_aligned ? 16 : mmsize;
532 2565 int8_t *mask = av_malloc(mask_size);
533
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2565 times.
 2565 if (!mask)
534 ✗ return AVERROR(ENOMEM);
535
536 2565 const int groups = ff_sws_shuffle_mask(uop, mask, mask_size);
537
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2565 times.
 2565 if (groups < 0) {
538 ✗ av_free(mask);
539 ✗ return groups;
540 }
541
542 2565 const int read_chunk = groups * par->read_size;
543 2565 const int write_chunk = groups * par->write_size;
544
2/2
✓ Branch 0 taken 1543 times.
✓ Branch 1 taken 1022 times.
 2565 const int num_lanes = lane_aligned ? mmsize / 16 : 1;
545 2565 const int in_total = num_lanes * read_chunk;
546 2565 const int out_total = num_lanes * write_chunk;
547 2565 *out = (SwsCompiledOp) {
548 .priv = mask,
549 .free = av_free,
550 .slice_align = 1,
551 2565 .block_size = groups * uop->data.shuffle.pixels * num_lanes,
552 2565 .over_read = { movsize(in_total, mmsize) - in_total },
553 2565 .over_write = { movsize(out_total, mmsize) - out_total },
554 };
555
556 #define ASSIGN_SHUFFLE_FUNC(CPU, EXT, NAME, ...) \
557 do { \
558 const SwsUOpEntry *entry = &uop_##NAME##EXT; \
559 if (!memcmp(&uop->par, &entry->par, sizeof(uop->par))) { \
560 out->func = (SwsOpFunc) entry->func; \
561 out->cpu_flags = AV_CPU_FLAG_##CPU; \
562 } \
563 } while (0);
564
565
2/4
✓ Branch 0 taken 1022 times.
✓ Branch 1 taken 1543 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2565 switch (mmsize) {
566
42/44
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 1018 times.
✓ Branch 2 taken 16 times.
✓ Branch 3 taken 1006 times.
✓ Branch 4 taken 8 times.
✓ Branch 5 taken 1014 times.
✓ Branch 6 taken 2 times.
✓ Branch 7 taken 1020 times.
✓ Branch 8 taken 8 times.
✓ Branch 9 taken 1014 times.
✓ Branch 10 taken 44 times.
✓ Branch 11 taken 978 times.
✓ Branch 12 taken 42 times.
✓ Branch 13 taken 980 times.
✓ Branch 14 taken 2 times.
✓ Branch 15 taken 1020 times.
✓ Branch 16 taken 14 times.
✓ Branch 17 taken 1008 times.
✓ Branch 18 taken 9 times.
✓ Branch 19 taken 1013 times.
✓ Branch 20 taken 20 times.
✓ Branch 21 taken 1002 times.
✓ Branch 22 taken 2 times.
✓ Branch 23 taken 1020 times.
✓ Branch 24 taken 187 times.
✓ Branch 25 taken 835 times.
✓ Branch 26 taken 547 times.
✓ Branch 27 taken 475 times.
✓ Branch 28 taken 40 times.
✓ Branch 29 taken 982 times.
✗ Branch 30 not taken.
✓ Branch 31 taken 1022 times.
✓ Branch 32 taken 4 times.
✓ Branch 33 taken 1018 times.
✓ Branch 34 taken 16 times.
✓ Branch 35 taken 1006 times.
✓ Branch 36 taken 8 times.
✓ Branch 37 taken 1014 times.
✓ Branch 38 taken 22 times.
✓ Branch 39 taken 1000 times.
✓ Branch 40 taken 27 times.
✓ Branch 41 taken 995 times.
✗ Branch 42 not taken.
✓ Branch 43 taken 1022 times.
 1022 case 16: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, SSE4, _sse4); break;
567
24/44
✗ Branch 0 not taken.
✓ Branch 1 taken 1543 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1543 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1543 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 1543 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 1543 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 1543 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 1543 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 1543 times.
✗ Branch 16 not taken.
✓ Branch 17 taken 1543 times.
✗ Branch 18 not taken.
✓ Branch 19 taken 1543 times.
✗ Branch 20 not taken.
✓ Branch 21 taken 1543 times.
✗ Branch 22 not taken.
✓ Branch 23 taken 1543 times.
✗ Branch 24 not taken.
✓ Branch 25 taken 1543 times.
✗ Branch 26 not taken.
✓ Branch 27 taken 1543 times.
✗ Branch 28 not taken.
✓ Branch 29 taken 1543 times.
✓ Branch 30 taken 1520 times.
✓ Branch 31 taken 23 times.
✗ Branch 32 not taken.
✓ Branch 33 taken 1543 times.
✗ Branch 34 not taken.
✓ Branch 35 taken 1543 times.
✗ Branch 36 not taken.
✓ Branch 37 taken 1543 times.
✗ Branch 38 not taken.
✓ Branch 39 taken 1543 times.
✗ Branch 40 not taken.
✓ Branch 41 taken 1543 times.
✓ Branch 42 taken 23 times.
✓ Branch 43 taken 1520 times.
 1543 case 32: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, AVX2, _avx2); break;
568 ✗ case 64:
569 ✗ if (lane_aligned) {
570 ✗ SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, AVX512, _avx512);
571 } else { /* vpermb variant */
572 ✗ SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, AVX512ICL, _avx512icl);
573 }
574 ✗ break;
575 }
576
577
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2565 times.
 2565 if (!out->func) {
578 ✗ av_free(mask);
579 ✗ return AVERROR(ENOTSUP);
580 }
581
582 2565 return 0;
583 }
584
585 /* Expand pixel value to 32-bits by repeating as necessary */
586 42772 static uint32_t expand32(const SwsPixelType type, const SwsPixel value)
587 {
588
3/5
✓ Branch 0 taken 12200 times.
✓ Branch 1 taken 24956 times.
✓ Branch 2 taken 5616 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 42772 switch (type) {
589 12200 case SWS_PIXEL_U8: return value.u8 * 0x01010101u;
590 24956 case SWS_PIXEL_U16: return value.u16 * 0x00010001u;
591 5616 case SWS_PIXEL_U32: return value.u32;
592 ✗ case SWS_PIXEL_F32: return value.u32; /* reinterpret */
593 ✗ default: return 0;
594 }
595 }
596
597 10693 static void normalize_clear(SwsUOp *uop)
598 {
599
2/2
✓ Branch 0 taken 42772 times.
✓ Branch 1 taken 10693 times.
 53465 for (int i = 0; i < 4; i++)
600 42772 uop->data.vec4[i].u32 = expand32(uop->type, uop->data.vec4[i]);
601 10693 }
602
603 89267 static int compile_uops_x86(SwsContext *ctx, const SwsUOpList *uops, SwsCompiledOp *out)
604 {
605 89267 int ret, mmsize = get_mmsize();
606
2/2
✓ Branch 0 taken 4417 times.
✓ Branch 1 taken 84850 times.
 89267 if (mmsize < 0)
607 4417 return mmsize;
608
609
3/4
✓ Branch 0 taken 2565 times.
✓ Branch 1 taken 82285 times.
✓ Branch 2 taken 2565 times.
✗ Branch 3 not taken.
 84850 if (uops->num_ops == 1 && uops->ops[0].uop == SWS_UOP_RW_SHUFFLE) {
610 2565 const SwsUOp *uop = &uops->ops[0];
611 2565 ret = translate_shuffle(uop, mmsize, out);
612
1/2
✓ Branch 0 taken 2565 times.
✗ Branch 1 not taken.
 2565 if (ret >= 0) {
613 char name[SWS_UOP_NAME_MAX];
614 2565 ff_sws_uop_name(uop, name);
615 2565 av_log(ctx, AV_LOG_VERBOSE, "Using x86 packed shuffle fast path: %s\n", name);
616 }
617 2565 return ret;
618 }
619
620 82285 SwsOpChain *chain = ff_sws_op_chain_alloc();
621
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 82285 times.
 82285 if (!chain)
622 ✗ return AVERROR(ENOMEM);
623
624 82285 *out = (SwsCompiledOp) {
625 /* Use at most two full YMM regs during the widest precision section */
626 82285 .block_size = 2 * FFMIN(mmsize, 32) / uops->pixel_size_max,
627 .slice_align = 1,
628 .free = ff_sws_op_chain_free_cb,
629 .priv = chain,
630 };
631
632
2/2
✓ Branch 0 taken 408828 times.
✓ Branch 1 taken 71057 times.
 479885 for (int i = 0; i < uops->num_ops; i++) {
633 408828 SwsUOp *uop = &uops->ops[i];
634 408828 int op_block_size = out->block_size;
635
636
2/2
✓ Branch 1 taken 95751 times.
✓ Branch 2 taken 313077 times.
 408828 if (uop_is_type_invariant(uop->uop)) {
637
2/2
✓ Branch 0 taken 10693 times.
✓ Branch 1 taken 85058 times.
 95751 if (uop->uop == SWS_UOP_CLEAR)
638 10693 normalize_clear(uop);
639 95751 op_block_size *= ff_sws_pixel_type_size(uop->type);
640 95751 uop->type = SWS_PIXEL_U8;
641 }
642
643 408828 ret = ff_sws_uop_lookup(ctx, tables, FF_ARRAY_ELEMS(tables), uop,
644 op_block_size, chain);
645
2/2
✓ Branch 0 taken 11228 times.
✓ Branch 1 taken 397600 times.
 408828 if (ret < 0)
646 11228 goto fail;
647 }
648
649
4/5
✓ Branch 0 taken 11588 times.
✓ Branch 1 taken 2758 times.
✓ Branch 2 taken 48314 times.
✓ Branch 3 taken 8397 times.
✗ Branch 4 not taken.
 71057 switch (av_popcount(uops->planes_in | uops->planes_out)) {
650 11588 case 1: out->func = ff_sws_process1_x86; break;
651 2758 case 2: out->func = ff_sws_process2_x86; break;
652 48314 case 3: out->func = ff_sws_process3_x86; break;
653 8397 case 4: out->func = ff_sws_process4_x86; break;
654 }
655
656
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 71057 times.
 71057 if (ret < 0) {
657 ✗ ff_sws_op_chain_free(chain);
658 ✗ return ret;
659 }
660
661 71057 out->cpu_flags = chain->cpu_flags;
662 71057 memcpy(out->over_read, chain->over_read, sizeof(out->over_read));
663 71057 memcpy(out->over_write, chain->over_write, sizeof(out->over_write));
664
665 71057 av_log(ctx, AV_LOG_DEBUG, "Compiled micro-ops:\n");
666
2/2
✓ Branch 0 taken 376106 times.
✓ Branch 1 taken 71057 times.
 447163 for (int i = 0; i < uops->num_ops; i++) {
667 char name[SWS_UOP_NAME_MAX];
668 376106 ff_sws_uop_name(&uops->ops[i], name);
669 376106 av_log(ctx, AV_LOG_DEBUG, " %s\n", name);
670 }
671
672 71057 return 0;
673
674 11228 fail:
675 11228 ff_sws_op_chain_free(chain);
676 11228 return ret;
677 }
678
679 112427 static int compile_x86(SwsContext *ctx, const SwsOpList *ops, SwsCompiledOp *out)
680 {
681 112427 const int cpu_flags = av_get_cpu_flags();
682 112427 const int mmsize = get_mmsize();
683
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 112427 times.
 112427 if (mmsize < 0)
684 ✗ return mmsize;
685
686 112427 SwsUOpFlags flags = SWS_UOP_FLAG_PSHUFB;
687
1/2
✓ Branch 0 taken 112427 times.
✗ Branch 1 not taken.
 112427 if (EXTERNAL_FMA3(cpu_flags))
688 112427 flags |= SWS_UOP_FLAG_FMA;
689
690 112427 SwsUOpList *uops = ff_sws_uop_list_alloc();
691
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 112427 times.
 112427 if (!uops)
692 ✗ return AVERROR(ENOMEM);
693
694 112427 int ret = ff_sws_ops_translate(ctx, ops, flags, uops);
695
2/2
✓ Branch 0 taken 31994 times.
✓ Branch 1 taken 80433 times.
 112427 if (ret < 0)
696 31994 goto fail;
697
698 80433 ret = compile_uops_x86(ctx, uops, out);
699
700 112427 fail:
701 112427 ff_sws_uop_list_free(&uops);
702 112427 return ret;
703 }
704
705 const SwsOpBackend backend_x86 = {
706 .name = "x86",
707 .flags = SWS_BACKEND_X86,
708 .compile = compile_x86,
709 .compile_uops = compile_uops_x86,
710 .hw_format = AV_PIX_FMT_NONE,
711 };
712