FFmpeg coverage

Directory: ../../../ffmpeg/
File: src/libswscale/uops_tmpl.c
Date: 2026-10-29 23:19:09
Exec Total Coverage
Lines: 489 503 97.2%
Functions: 474 482 98.3%
Branches: 391 464 84.3%
Line Branch Exec Source
1 /**
2 * Copyright (C) 2026 Niklas Haas
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include <libavutil/bswap.h>
22
23 #include "uops_tmpl.h"
24
25 #ifndef BIT_DEPTH
26 # define BIT_DEPTH 8
27 #endif
28
29 #if IS_FLOAT && BIT_DEPTH == 32
30 # define PIXEL_TYPE SWS_PIXEL_F32
31 # define pixel_t float
32 # define inter_t float
33 # define uinter_t float
34 # define vec3_t v3f32_t
35 # define PX F32
36 # define px f32
37 #elif BIT_DEPTH == 32
38 # define PIXEL_MAX 0xFFFFFFFFu
39 # define PIXEL_SWAP av_bswap32
40 # define pixel_t uint32_t
41 # define inter_t int64_t
42 # define uinter_t uint64_t
43 # define PX U32
44 # define px u32
45 #elif BIT_DEPTH == 16
46 # define PIXEL_MAX 0xFFFFu
47 # define PIXEL_SWAP av_bswap16
48 # define pixel_t uint16_t
49 # define inter_t int64_t
50 # define uinter_t uint64_t
51 # define PX U16
52 # define px u16
53 #elif BIT_DEPTH == 8
54 # define PIXEL_MAX 0xFFu
55 # define pixel_t uint8_t
56 # define inter_t int32_t
57 # define uinter_t uint32_t
58 # define PX U8
59 # define px u8
60 #else
61 # error Invalid BIT_DEPTH
62 #endif
63
64 /*********************************
65 * Generic read/write operations *
66 *********************************/
67
68 20531008 DECL_READ(read_planar, const SwsCompMask mask)
69 {
70 SWS_LOOP
71
2/2
✓ Branch 0 taken 328496128 times.
✓ Branch 1 taken 10265504 times.
 677523264 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
72
1/2
✓ Branch 0 taken 328496128 times.
✗ Branch 1 not taken.
 656992256 if (X) x[i] = in0[i];
73
2/2
✓ Branch 0 taken 161634304 times.
✓ Branch 1 taken 166861824 times.
 656992256 if (Y) y[i] = in1[i];
74
2/2
✓ Branch 0 taken 160518144 times.
✓ Branch 1 taken 167977984 times.
 656992256 if (Z) z[i] = in2[i];
75
2/2
✓ Branch 0 taken 22852608 times.
✓ Branch 1 taken 305643520 times.
 656992256 if (W) w[i] = in3[i];
76 }
77
78
1/2
✓ Branch 0 taken 10265504 times.
✗ Branch 1 not taken.
 20531008 if (X) iter->in[0] += SIZEOF_BLOCK;
79
2/2
✓ Branch 0 taken 5051072 times.
✓ Branch 1 taken 5214432 times.
 20531008 if (Y) iter->in[1] += SIZEOF_BLOCK;
80
2/2
✓ Branch 0 taken 5016192 times.
✓ Branch 1 taken 5249312 times.
 20531008 if (Z) iter->in[2] += SIZEOF_BLOCK;
81
2/2
✓ Branch 0 taken 714144 times.
✓ Branch 1 taken 9551360 times.
 20531008 if (W) iter->in[3] += SIZEOF_BLOCK;
82
83 20531008 CONTINUE(x, y, z, w);
84 20531008 }
85
86 11438976 DECL_READ(read_packed, const SwsCompMask mask)
87 {
88
5/6
✓ Branch 0 taken 2125696 times.
✓ Branch 1 taken 3593792 times.
✓ Branch 2 taken 774848 times.
✓ Branch 3 taken 1350848 times.
✓ Branch 4 taken 774848 times.
✗ Branch 5 not taken.
 11438976 const int elems = W ? 4 : Z ? 3 : Y ? 2 : 1;
89
90 SWS_LOOP
91
2/2
✓ Branch 0 taken 183023616 times.
✓ Branch 1 taken 5719488 times.
 377486208 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
92
1/2
✓ Branch 0 taken 183023616 times.
✗ Branch 1 not taken.
 366047232 if (X) x[i] = in0[elems * i + 0];
93
1/2
✓ Branch 0 taken 183023616 times.
✗ Branch 1 not taken.
 366047232 if (Y) y[i] = in0[elems * i + 1];
94
2/2
✓ Branch 0 taken 158228480 times.
✓ Branch 1 taken 24795136 times.
 366047232 if (Z) z[i] = in0[elems * i + 2];
95
2/2
✓ Branch 0 taken 115001344 times.
✓ Branch 1 taken 68022272 times.
 366047232 if (W) w[i] = in0[elems * i + 3];
96 }
97
98 11438976 iter->in[0] += SIZEOF_BLOCK * elems;
99 11438976 CONTINUE(x, y, z, w);
100 11438976 }
101
102 64140288 DECL_WRITE(write_planar, const SwsCompMask mask)
103 {
104 SWS_LOOP
105
2/2
✓ Branch 0 taken 1026244608 times.
✓ Branch 1 taken 32070144 times.
 2116629504 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
106
1/2
✓ Branch 0 taken 1026244608 times.
✗ Branch 1 not taken.
 2052489216 if (X) out0[i] = x[i];
107
2/2
✓ Branch 0 taken 695377920 times.
✓ Branch 1 taken 330866688 times.
 2052489216 if (Y) out1[i] = y[i];
108
2/2
✓ Branch 0 taken 670030848 times.
✓ Branch 1 taken 356213760 times.
 2052489216 if (Z) out2[i] = z[i];
109
2/2
✓ Branch 0 taken 79975424 times.
✓ Branch 1 taken 946269184 times.
 2052489216 if (W) out3[i] = w[i];
110 }
111
112
1/2
✓ Branch 0 taken 32070144 times.
✗ Branch 1 not taken.
 64140288 if (X) iter->out[0] += SIZEOF_BLOCK;
113
2/2
✓ Branch 0 taken 21730560 times.
✓ Branch 1 taken 10339584 times.
 64140288 if (Y) iter->out[1] += SIZEOF_BLOCK;
114
2/2
✓ Branch 0 taken 20938464 times.
✓ Branch 1 taken 11131680 times.
 64140288 if (Z) iter->out[2] += SIZEOF_BLOCK;
115
2/2
✓ Branch 0 taken 2499232 times.
✓ Branch 1 taken 29570912 times.
 64140288 if (W) iter->out[3] += SIZEOF_BLOCK;
116 64140288 }
117
118 11196672 DECL_WRITE(write_packed, const SwsCompMask mask)
119 {
120
5/6
✓ Branch 0 taken 2062656 times.
✓ Branch 1 taken 3535680 times.
✓ Branch 2 taken 755072 times.
✓ Branch 3 taken 1307584 times.
✓ Branch 4 taken 755072 times.
✗ Branch 5 not taken.
 11196672 const int elems = W ? 4 : Z ? 3 : Y ? 2 : 1;
121
122 SWS_LOOP
123
2/2
✓ Branch 0 taken 179146752 times.
✓ Branch 1 taken 5598336 times.
 369490176 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
124
1/2
✓ Branch 0 taken 179146752 times.
✗ Branch 1 not taken.
 358293504 if (X) out0[elems * i + 0] = x[i];
125
1/2
✓ Branch 0 taken 179146752 times.
✗ Branch 1 not taken.
 358293504 if (Y) out0[elems * i + 1] = y[i];
126
2/2
✓ Branch 0 taken 154984448 times.
✓ Branch 1 taken 24162304 times.
 358293504 if (Z) out0[elems * i + 2] = z[i];
127
2/2
✓ Branch 0 taken 113141760 times.
✓ Branch 1 taken 66004992 times.
 358293504 if (W) out0[elems * i + 3] = w[i];
128 }
129
130 11196672 iter->out[0] += SIZEOF_BLOCK * elems;
131 11196672 }
132
133 #if BIT_DEPTH == 8
134
135 290176 DECL_READ(read_bit, const SwsCompMask mask)
136 {
137 av_assert2(mask == SWS_COMP_ELEMS(1));
138
139 SWS_LOOP
140
2/2
✓ Branch 0 taken 1160704 times.
✓ Branch 1 taken 290176 times.
 1450880 for (int i = 0; i < SWS_BLOCK_SIZE; i += 8) {
141 1160704 const pixel_t val = ((const pixel_t *) in0)[i >> 3];
142 1160704 x[i + 0] = (val >> 7) & 1;
143 1160704 x[i + 1] = (val >> 6) & 1;
144 1160704 x[i + 2] = (val >> 5) & 1;
145 1160704 x[i + 3] = (val >> 4) & 1;
146 1160704 x[i + 4] = (val >> 3) & 1;
147 1160704 x[i + 5] = (val >> 2) & 1;
148 1160704 x[i + 6] = (val >> 1) & 1;
149 1160704 x[i + 7] = (val >> 0) & 1;
150 }
151
152 290176 iter->in[0] += SIZEOF_BLOCK >> 3;
153 290176 CONTINUE(x, y, z, w);
154 290176 }
155
156 277792 DECL_READ(read_nibble, const SwsCompMask mask)
157 {
158 av_assert2(mask == SWS_COMP_ELEMS(1));
159
160 SWS_LOOP
161
2/2
✓ Branch 0 taken 4444672 times.
✓ Branch 1 taken 277792 times.
 4722464 for (int i = 0; i < SWS_BLOCK_SIZE; i += 2) {
162 4444672 const pixel_t val = in0[i >> 1];
163 4444672 x[i + 0] = val >> 4; /* high nibble */
164 4444672 x[i + 1] = val & 0xF; /* low nibble */
165 }
166
167 277792 iter->in[0] += SIZEOF_BLOCK >> 1;
168 277792 CONTINUE(x, y, z, w);
169 277792 }
170
171 160 DECL_READ(read_palette, const SwsCompMask mask)
172 {
173 av_assert2(mask == SWS_COMP_ELEMS(4));
174
175 SWS_LOOP
176
2/2
✓ Branch 0 taken 5120 times.
✓ Branch 1 taken 160 times.
 5280 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
177 5120 const pixel_t index = in0[i];
178 5120 const pixel_t *value = &in1[index * 4];
179 5120 x[i] = value[0];
180 5120 y[i] = value[1];
181 5120 z[i] = value[2];
182 5120 w[i] = value[3];
183 }
184
185 160 iter->in[0] += SIZEOF_BLOCK;
186 160 CONTINUE(x, y, z, w);
187 160 }
188
189 300672 DECL_WRITE(write_bit, const SwsCompMask mask)
190 {
191 av_assert2(mask == SWS_COMP_ELEMS(1));
192
193 SWS_LOOP
194
2/2
✓ Branch 0 taken 1202688 times.
✓ Branch 1 taken 300672 times.
 1503360 for (int i = 0; i < SWS_BLOCK_SIZE; i += 8) {
195 1202688 out0[i >> 3] = x[i + 0] << 7 |
196 1202688 x[i + 1] << 6 |
197 1202688 x[i + 2] << 5 |
198 1202688 x[i + 3] << 4 |
199 1202688 x[i + 4] << 3 |
200 1202688 x[i + 5] << 2 |
201 1202688 x[i + 6] << 1 |
202 1202688 x[i + 7];
203 }
204
205 300672 iter->out[0] += SIZEOF_BLOCK >> 3;
206 300672 }
207
208 300352 DECL_WRITE(write_nibble, const SwsCompMask mask)
209 {
210 av_assert2(mask == SWS_COMP_ELEMS(1));
211
212 SWS_LOOP
213
2/2
✓ Branch 0 taken 4805632 times.
✓ Branch 1 taken 300352 times.
 5105984 for (int i = 0; i < SWS_BLOCK_SIZE; i += 2)
214 4805632 out0[i >> 1] = x[i] << 4 | x[i + 1];
215
216 300352 iter->out[0] += SIZEOF_BLOCK >> 1;
217 300352 }
218
219 #endif /* BIT_DEPTH == 8 */
220
221 20531008 SWS_FOR(PX, READ_PLANAR, DECL_IMPL_READ, read_planar)
222 11438976 SWS_FOR(PX, READ_PACKED, DECL_IMPL_READ, read_packed)
223 277792 SWS_FOR(PX, READ_NIBBLE, DECL_IMPL_READ, read_nibble)
224 290176 SWS_FOR(PX, READ_BIT, DECL_IMPL_READ, read_bit)
225 160 SWS_FOR(PX, READ_PALETTE, DECL_IMPL_READ, read_palette)
226 64140288 SWS_FOR(PX, WRITE_PLANAR, DECL_IMPL_WRITE, write_planar)
227 11196672 SWS_FOR(PX, WRITE_PACKED, DECL_IMPL_WRITE, write_packed)
228 300352 SWS_FOR(PX, WRITE_NIBBLE, DECL_IMPL_WRITE, write_nibble)
229 300672 SWS_FOR(PX, WRITE_BIT, DECL_IMPL_WRITE, write_bit)
230
231 SWS_FOR_STRUCT(PX, READ_PLANAR, DECL_ENTRY)
232 SWS_FOR_STRUCT(PX, READ_PACKED, DECL_ENTRY)
233 SWS_FOR_STRUCT(PX, READ_NIBBLE, DECL_ENTRY)
234 SWS_FOR_STRUCT(PX, READ_BIT, DECL_ENTRY)
235 SWS_FOR_STRUCT(PX, READ_PALETTE, DECL_ENTRY)
236 SWS_FOR_STRUCT(PX, WRITE_PLANAR, DECL_ENTRY)
237 SWS_FOR_STRUCT(PX, WRITE_PACKED, DECL_ENTRY)
238 SWS_FOR_STRUCT(PX, WRITE_NIBBLE, DECL_ENTRY)
239 SWS_FOR_STRUCT(PX, WRITE_BIT, DECL_ENTRY)
240
241 /*****************************
242 * Scaling / filtering reads *
243 *****************************/
244
245 70656 DECL_SETUP(setup_filter_v, params, out)
246 {
247
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35328 times.
 70656 if (params->uop->par.filter.type != SWS_PIXEL_F32)
248 ✗ return AVERROR(ENOTSUP);
249
250 70656 const SwsFilterWeights *filter = params->uop->data.kernel;
251 static_assert(sizeof(out->priv.ptr) <= sizeof(int32_t[2]),
252 ">8 byte pointers not supported");
253
254 /* Pre-convert weights to float */
255 70656 float *weights = av_calloc(filter->num_weights, sizeof(float));
256
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35328 times.
 70656 if (!weights)
257 ✗ return AVERROR(ENOMEM);
258
259
2/2
✓ Branch 0 taken 11562624 times.
✓ Branch 1 taken 35328 times.
 23195904 for (int i = 0; i < filter->num_weights; i++)
260 23125248 weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE;
261
262 70656 out->priv.ptr = weights;
263 70656 out->priv.i32[2] = filter->filter_size;
264 70656 out->free = ff_op_priv_free;
265 70656 return 0;
266 }
267
268 /* Fully general vertical planar filter case */
269 19330048 DECL_READ(read_planar_fv, const SwsCompMask mask, const SwsPixelType type)
270 {
271 av_assert2(type == SWS_PIXEL_F32);
272 19330048 const SwsOpExec *exec = iter->exec;
273 19330048 const float *restrict weights = impl->priv.ptr;
274 19330048 const int filter_size = impl->priv.i32[2];
275 19330048 weights += filter_size * iter->y;
276
277 block_t xs, ys, zs, ws;
278
1/2
✓ Branch 0 taken 9665024 times.
✗ Branch 1 not taken.
 19330048 if (X) memset(&xs.f32, 0, sizeof(xs.f32));
279
2/2
✓ Branch 0 taken 7927616 times.
✓ Branch 1 taken 1737408 times.
 19330048 if (Y) memset(&ys.f32, 0, sizeof(ys.f32));
280
2/2
✓ Branch 0 taken 7743808 times.
✓ Branch 1 taken 1921216 times.
 19330048 if (Z) memset(&zs.f32, 0, sizeof(zs.f32));
281
2/2
✓ Branch 0 taken 897984 times.
✓ Branch 1 taken 8767040 times.
 19330048 if (W) memset(&ws.f32, 0, sizeof(ws.f32));
282
283
2/2
✓ Branch 0 taken 34372992 times.
✓ Branch 1 taken 9665024 times.
 88076032 for (int j = 0; j < filter_size; j++) {
284 68745984 const float weight = weights[j];
285
286 SWS_LOOP
287
2/2
✓ Branch 0 taken 1099935744 times.
✓ Branch 1 taken 34372992 times.
 2268617472 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
288
1/2
✓ Branch 0 taken 1099935744 times.
✗ Branch 1 not taken.
 2199871488 if (X) xs.f32[i] += weight * in0[i];
289
2/2
✓ Branch 0 taken 902006784 times.
✓ Branch 1 taken 197928960 times.
 2199871488 if (Y) ys.f32[i] += weight * in1[i];
290
2/2
✓ Branch 0 taken 881551360 times.
✓ Branch 1 taken 218384384 times.
 2199871488 if (Z) zs.f32[i] += weight * in2[i];
291
2/2
✓ Branch 0 taken 104288256 times.
✓ Branch 1 taken 995647488 times.
 2199871488 if (W) ws.f32[i] += weight * in3[i];
292 }
293
294
1/2
✓ Branch 0 taken 34372992 times.
✗ Branch 1 not taken.
 68745984 if (X) in0 = bump_ptr(in0, exec->in_stride[0]);
295
2/2
✓ Branch 0 taken 28187712 times.
✓ Branch 1 taken 6185280 times.
 68745984 if (Y) in1 = bump_ptr(in1, exec->in_stride[1]);
296
2/2
✓ Branch 0 taken 27548480 times.
✓ Branch 1 taken 6824512 times.
 68745984 if (Z) in2 = bump_ptr(in2, exec->in_stride[2]);
297
2/2
✓ Branch 0 taken 3259008 times.
✓ Branch 1 taken 31113984 times.
 68745984 if (W) in3 = bump_ptr(in3, exec->in_stride[3]);
298 }
299
300
1/2
✓ Branch 0 taken 9665024 times.
✗ Branch 1 not taken.
 19330048 if (X) iter->in[0] += SIZEOF_BLOCK;
301
2/2
✓ Branch 0 taken 7927616 times.
✓ Branch 1 taken 1737408 times.
 19330048 if (Y) iter->in[1] += SIZEOF_BLOCK;
302
2/2
✓ Branch 0 taken 7743808 times.
✓ Branch 1 taken 1921216 times.
 19330048 if (Z) iter->in[2] += SIZEOF_BLOCK;
303
2/2
✓ Branch 0 taken 897984 times.
✓ Branch 1 taken 8767040 times.
 19330048 if (W) iter->in[3] += SIZEOF_BLOCK;
304
305 19330048 CONTINUE(&xs, &ys, &zs, &ws);
306 19330048 }
307
308 71980 DECL_SETUP(setup_filter_h, params, out)
309 {
310
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 35990 times.
 71980 if (params->uop->par.filter.type != SWS_PIXEL_F32)
311 ✗ return AVERROR(ENOTSUP);
312
313 71980 SwsFilterWeights *filter = params->uop->data.kernel;
314 71980 out->priv.i32[2] = filter->filter_size;
315
316 /* The horizontal filter uop reads weights for a full SWS_BLOCK_SIZE
317 * outputs per block (see read_planar_fh), so the weights array must be
318 * padded up to the block-aligned output count. Pad the tail with zero
319 * weights, which contribute nothing to the accumulated sums. */
320 71980 const size_t padded_w = FFALIGN(filter->dst_size, SWS_BLOCK_SIZE);
321
1/2
✓ Branch 0 taken 35990 times.
✗ Branch 1 not taken.
 71980 if (padded_w == filter->dst_size) {
322 71980 out->priv.ptr = av_refstruct_ref(filter->weights);
323 71980 out->free = ff_op_priv_unref;
324 71980 return 0;
325 }
326
327 ✗ int *weights = av_calloc(padded_w * filter->filter_size, sizeof(*weights));
328 ✗ if (!weights)
329 ✗ return AVERROR(ENOMEM);
330 ✗ memcpy(weights, filter->weights, filter->num_weights * sizeof(*weights));
331 ✗ out->priv.ptr = weights;
332 ✗ out->free = ff_op_priv_free;
333 ✗ return 0;
334 }
335
336 /* Fully general horizontal planar filter case */
337 19411584 DECL_READ(read_planar_fh, const SwsCompMask mask, const SwsPixelType type)
338 {
339 av_assert2(type == SWS_PIXEL_F32);
340 19411584 const SwsOpExec *exec = iter->exec;
341 19411584 const int *restrict weights = impl->priv.ptr;
342 19411584 const int filter_size = impl->priv.i32[2];
343 19411584 const float scale = 1.0f / SWS_FILTER_SCALE;
344 19411584 const int xpos = iter->x;
345 19411584 weights += filter_size * iter->x;
346
347 block_t xs, ys, zs, ws;
348
2/2
✓ Branch 0 taken 310585344 times.
✓ Branch 1 taken 9705792 times.
 640582272 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
349 621170688 const int offset = exec->in_offset_x[xpos + i];
350 621170688 pixel_t *start0 = bump_ptr(in0, offset);
351 621170688 pixel_t *start1 = bump_ptr(in1, offset);
352 621170688 pixel_t *start2 = bump_ptr(in2, offset);
353 621170688 pixel_t *start3 = bump_ptr(in3, offset);
354
355 621170688 inter_t sx = 0, sy = 0, sz = 0, sw = 0;
356
2/2
✓ Branch 0 taken 1110847488 times.
✓ Branch 1 taken 310585344 times.
 2842865664 for (int j = 0; j < filter_size; j++) {
357 2221694976 const int weight = weights[j];
358
1/2
✓ Branch 0 taken 1110847488 times.
✗ Branch 1 not taken.
 2221694976 if (X) sx += weight * start0[j];
359
2/2
✓ Branch 0 taken 912574464 times.
✓ Branch 1 taken 198273024 times.
 2221694976 if (Y) sy += weight * start1[j];
360
2/2
✓ Branch 0 taken 891101184 times.
✓ Branch 1 taken 219746304 times.
 2221694976 if (Z) sz += weight * start2[j];
361
2/2
✓ Branch 0 taken 107599872 times.
✓ Branch 1 taken 1003247616 times.
 2221694976 if (W) sw += weight * start3[j];
362 }
363
364
1/2
✓ Branch 0 taken 310585344 times.
✗ Branch 1 not taken.
 621170688 if (X) xs.f32[i] = (float) sx * scale;
365
2/2
✓ Branch 0 taken 255823872 times.
✓ Branch 1 taken 54761472 times.
 621170688 if (Y) ys.f32[i] = (float) sy * scale;
366
2/2
✓ Branch 0 taken 249821184 times.
✓ Branch 1 taken 60764160 times.
 621170688 if (Z) zs.f32[i] = (float) sz * scale;
367
2/2
✓ Branch 0 taken 29988864 times.
✓ Branch 1 taken 280596480 times.
 621170688 if (W) ws.f32[i] = (float) sw * scale;
368
369 621170688 weights += filter_size;
370 }
371
372 19411584 CONTINUE(&xs, &ys, &zs, &ws);
373 19411584 }
374
375 19330048 SWS_FOR(PX, READ_PLANAR_FV, DECL_IMPL_READ, read_planar_fv)
376 19411584 SWS_FOR(PX, READ_PLANAR_FH, DECL_IMPL_READ, read_planar_fh)
377 SWS_FOR_STRUCT(PX, READ_PLANAR_FV, DECL_ENTRY, .setup = fn(setup_filter_v) )
378 SWS_FOR_STRUCT(PX, READ_PLANAR_FH, DECL_ENTRY, .setup = fn(setup_filter_h) )
379
380 /***************************
381 * Permutation and copying *
382 ***************************/
383
384 18746112 DECL_FUNC(permute, const SwsCompMask mask, int num_moves,
385 int8_t d0, int8_t d1, int8_t d2, int8_t d3, int8_t d4, int8_t d5,
386 int8_t s0, int8_t s1, int8_t s2, int8_t s3, int8_t s4, int8_t s5)
387 {
388 18746112 const int8_t dst[SWS_UOP_MOVE_MAX] = { d0, d1, d2, d3, d4, d5 };
389 18746112 const int8_t src[SWS_UOP_MOVE_MAX] = { s0, s1, s2, s3, s4, s5 };
390
391 18746112 pixel_t *ptr[5] = { NULL, x, y, z, w };
392
2/2
✓ Branch 0 taken 26373056 times.
✓ Branch 1 taken 9373056 times.
 71492224 for (int n = 0; n < num_moves; n++)
393 52746112 ptr[dst[n] + 1] = ptr[src[n] + 1];
394
395 /* The unneeded registers may still alias the used ones, so point them
396 * back at the stack to avoid collisions */
397 block_t xx, yy, zz, ww;
398
8/8
✓ Branch 0 taken 7429472 times.
✓ Branch 1 taken 1943584 times.
✓ Branch 2 taken 9279808 times.
✓ Branch 3 taken 93248 times.
✓ Branch 4 taken 8952512 times.
✓ Branch 5 taken 420544 times.
✓ Branch 6 taken 7897664 times.
✓ Branch 7 taken 1475392 times.
 18746112 CONTINUE(X ? ptr[1] : xx.px,
399 Y ? ptr[2] : yy.px,
400 Z ? ptr[3] : zz.px,
401 W ? ptr[4] : ww.px);
402 18746112 }
403
404 3205248 DECL_FUNC(copy, const SwsCompMask mask, int num_moves,
405 int8_t d0, int8_t d1, int8_t d2, int8_t d3, int8_t d4, int8_t d5,
406 int8_t s0, int8_t s1, int8_t s2, int8_t s3, int8_t s4, int8_t s5)
407 {
408 3205248 const size_t block_size = SWS_BLOCK_SIZE * sizeof(pixel_t);
409 3205248 const int8_t dst[SWS_UOP_MOVE_MAX] = { d0, d1, d2, d3, d4, d5 };
410 3205248 const int8_t src[SWS_UOP_MOVE_MAX] = { s0, s1, s2, s3, s4, s5 };
411
412 block_t data[5];
413 3205248 memcpy(&data[1].px, x, block_size);
414 3205248 memcpy(&data[2].px, y, block_size);
415 3205248 memcpy(&data[3].px, z, block_size);
416 3205248 memcpy(&data[4].px, w, block_size);
417
418
2/2
✓ Branch 0 taken 3509440 times.
✓ Branch 1 taken 1602624 times.
 10224128 for (int n = 0; n < num_moves; n++)
419 7018880 data[dst[n] + 1] = data[src[n] + 1];
420
421 3205248 memcpy(x, &data[1].px, block_size);
422 3205248 memcpy(y, &data[2].px, block_size);
423 3205248 memcpy(z, &data[3].px, block_size);
424 3205248 memcpy(w, &data[4].px, block_size);
425
426 3205248 CONTINUE(x, y, z, w);
427 3205248 }
428
429 18746112 SWS_FOR(PX, PERMUTE, DECL_IMPL, permute)
430 3205248 SWS_FOR(PX, COPY, DECL_IMPL, copy)
431 SWS_FOR_STRUCT(PX, PERMUTE, DECL_ENTRY)
432 SWS_FOR_STRUCT(PX, COPY, DECL_ENTRY)
433
434 /*********************
435 * Format conversion *
436 *********************/
437
438 #define DECL_CAST(DST, dst) \
439 DECL_FUNC(to_##dst, const SwsCompMask mask) \
440 { \
441 block_t xx, yy, zz, ww; \
442 \
443 SWS_LOOP \
444 for (int i = 0; i < SWS_BLOCK_SIZE; i++) { \
445 if (X) xx.dst[i] = x[i]; \
446 if (Y) yy.dst[i] = y[i]; \
447 if (Z) zz.dst[i] = z[i]; \
448 if (W) ww.dst[i] = w[i]; \
449 } \
450 \
451 CONTINUE(&xx, &yy, &zz, &ww); \
452 } \
453 \
454 SWS_FOR(PX, TO_##DST, DECL_IMPL, to_##dst) \
455 SWS_FOR_STRUCT(PX, TO_##DST, DECL_ENTRY)
456
457
10/10
✓ Branch 0 taken 180091904 times.
✓ Branch 1 taken 2151424 times.
✓ Branch 2 taken 149863424 times.
✓ Branch 3 taken 32379904 times.
✓ Branch 4 taken 148740096 times.
✓ Branch 5 taken 33503232 times.
✓ Branch 6 taken 15006720 times.
✓ Branch 7 taken 167236608 times.
✓ Branch 8 taken 182243328 times.
✓ Branch 9 taken 5695104 times.
 387267072 DECL_CAST(U8, u8)
458
10/10
✓ Branch 0 taken 372755456 times.
✓ Branch 1 taken 22544384 times.
✓ Branch 2 taken 302257152 times.
✓ Branch 3 taken 93042688 times.
✓ Branch 4 taken 300139520 times.
✓ Branch 5 taken 95160320 times.
✓ Branch 6 taken 53172224 times.
✓ Branch 7 taken 342127616 times.
✓ Branch 8 taken 395299840 times.
✓ Branch 9 taken 12353120 times.
 840012160 DECL_CAST(U16, u16)
459
10/10
✓ Branch 0 taken 37341184 times.
✓ Branch 1 taken 86016 times.
✓ Branch 2 taken 32311296 times.
✓ Branch 3 taken 5115904 times.
✓ Branch 4 taken 32311296 times.
✓ Branch 5 taken 5115904 times.
✓ Branch 6 taken 11753472 times.
✓ Branch 7 taken 25673728 times.
✓ Branch 8 taken 37427200 times.
✓ Branch 9 taken 1169600 times.
 79532800 DECL_CAST(U32, u32)
460
10/10
✓ Branch 0 taken 148938752 times.
✓ Branch 1 taken 23013376 times.
✓ Branch 2 taken 145989632 times.
✓ Branch 3 taken 25962496 times.
✓ Branch 4 taken 142125056 times.
✓ Branch 5 taken 29827072 times.
✓ Branch 6 taken 35251200 times.
✓ Branch 7 taken 136700928 times.
✓ Branch 8 taken 171952128 times.
✓ Branch 9 taken 5373504 times.
 365398272 DECL_CAST(F32, f32)
461
462 /********************
463 * Bit manipulation *
464 ********************/
465
466 #if !IS_FLOAT
467 3433920 DECL_FUNC(lshift, const SwsCompMask mask, const uint8_t amount)
468 {
469 SWS_LOOP
470
2/2
✓ Branch 0 taken 54942720 times.
✓ Branch 1 taken 1716960 times.
 113319360 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
471
2/2
✓ Branch 0 taken 53825536 times.
✓ Branch 1 taken 1117184 times.
 109885440 if (X) x[i] <<= amount;
472
2/2
✓ Branch 0 taken 47730688 times.
✓ Branch 1 taken 7212032 times.
 109885440 if (Y) y[i] <<= amount;
473
2/2
✓ Branch 0 taken 47730688 times.
✓ Branch 1 taken 7212032 times.
 109885440 if (Z) z[i] <<= amount;
474
2/2
✓ Branch 0 taken 1117184 times.
✓ Branch 1 taken 53825536 times.
 109885440 if (W) w[i] <<= amount;
475 }
476
477 3433920 CONTINUE(x, y, z, w);
478 3433920 }
479
480 1490176 DECL_FUNC(rshift, const SwsCompMask mask, const uint8_t amount)
481 {
482 SWS_LOOP
483
2/2
✓ Branch 0 taken 47685632 times.
✓ Branch 1 taken 1490176 times.
 49175808 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
484
2/2
✓ Branch 0 taken 46253056 times.
✓ Branch 1 taken 1432576 times.
 47685632 if (X) x[i] >>= amount;
485
2/2
✓ Branch 0 taken 44590080 times.
✓ Branch 1 taken 3095552 times.
 47685632 if (Y) y[i] >>= amount;
486
2/2
✓ Branch 0 taken 43157504 times.
✓ Branch 1 taken 4528128 times.
 47685632 if (Z) z[i] >>= amount;
487
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 47685632 times.
 47685632 if (W) w[i] >>= amount;
488 }
489
490 1490176 CONTINUE(x, y, z, w);
491 1490176 }
492 #endif
493
494 3433920 SWS_FOR(PX, LSHIFT, DECL_IMPL, lshift)
495 2980352 SWS_FOR(PX, RSHIFT, DECL_IMPL, rshift)
496
497 SWS_FOR_STRUCT(PX, LSHIFT, DECL_ENTRY)
498 SWS_FOR_STRUCT(PX, RSHIFT, DECL_ENTRY)
499
500 #ifdef PIXEL_SWAP
501 30186304 DECL_FUNC(swap_bytes, const SwsCompMask mask)
502 {
503 SWS_LOOP
504
2/2
✓ Branch 0 taken 482980864 times.
✓ Branch 1 taken 15093152 times.
 996148032 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
505
2/2
✓ Branch 0 taken 476250112 times.
✓ Branch 1 taken 6730752 times.
 965961728 if (X) x[i] = PIXEL_SWAP(x[i]);
506
2/2
✓ Branch 0 taken 288681984 times.
✓ Branch 1 taken 194298880 times.
 965961728 if (Y) y[i] = PIXEL_SWAP(y[i]);
507
2/2
✓ Branch 0 taken 276978688 times.
✓ Branch 1 taken 206002176 times.
 965961728 if (Z) z[i] = PIXEL_SWAP(z[i]);
508
2/2
✓ Branch 0 taken 52865024 times.
✓ Branch 1 taken 430115840 times.
 965961728 if (W) w[i] = PIXEL_SWAP(w[i]);
509 }
510
511 30186304 CONTINUE(x, y, z, w);
512 30186304 }
513 #endif /* PIXEL_SWAP */
514
515 30186304 SWS_FOR(PX, SWAP_BYTES, DECL_IMPL, swap_bytes)
516 SWS_FOR_STRUCT(PX, SWAP_BYTES, DECL_ENTRY)
517
518 #ifdef PIXEL_MAX
519 29760 DECL_FUNC(expand_bit, const SwsCompMask mask)
520 {
521 SWS_LOOP
522
2/2
✓ Branch 0 taken 476160 times.
✓ Branch 1 taken 14880 times.
 982080 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
523
3/4
✓ Branch 0 taken 476160 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 229087 times.
✓ Branch 3 taken 247073 times.
 952320 if (X) x[i] = x[i] ? PIXEL_MAX : 0;
524
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 476160 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 952320 if (Y) y[i] = y[i] ? PIXEL_MAX : 0;
525
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 476160 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 952320 if (Z) z[i] = z[i] ? PIXEL_MAX : 0;
526
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 476160 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 952320 if (W) w[i] = w[i] ? PIXEL_MAX : 0;
527 }
528
529 29760 CONTINUE(x, y, z, w);
530 29760 }
531 #endif
532
533 #if BIT_DEPTH == 8
534 68032 DECL_FUNC(expand_pair, const SwsCompMask mask)
535 {
536 block_t x16, y16, z16, w16;
537
538 SWS_LOOP
539
2/2
✓ Branch 0 taken 2177024 times.
✓ Branch 1 taken 68032 times.
 2245056 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
540
2/2
✓ Branch 0 taken 1739776 times.
✓ Branch 1 taken 437248 times.
 2177024 if (X) x16.u16[i] = x[i] << 8 | x[i];
541
2/2
✓ Branch 0 taken 1831936 times.
✓ Branch 1 taken 345088 times.
 2177024 if (Y) y16.u16[i] = y[i] << 8 | y[i];
542
2/2
✓ Branch 0 taken 1744896 times.
✓ Branch 1 taken 432128 times.
 2177024 if (Z) z16.u16[i] = z[i] << 8 | z[i];
543
2/2
✓ Branch 0 taken 819200 times.
✓ Branch 1 taken 1357824 times.
 2177024 if (W) w16.u16[i] = w[i] << 8 | w[i];
544 }
545
546 68032 CONTINUE(&x16, &y16, &z16, &w16);
547 68032 }
548
549 DECL_FUNC(expand_quad, const SwsCompMask mask)
550 {
551 block_t x32, y32, z32, w32;
552
553 SWS_LOOP
554 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
555 if (X) x32.u32[i] = (uint32_t) x[i] << 24 | x[i] << 16 | x[i] << 8 | x[i];
556 if (Y) y32.u32[i] = (uint32_t) y[i] << 24 | y[i] << 16 | y[i] << 8 | y[i];
557 if (Z) z32.u32[i] = (uint32_t) z[i] << 24 | z[i] << 16 | z[i] << 8 | z[i];
558 if (W) w32.u32[i] = (uint32_t) w[i] << 24 | w[i] << 16 | w[i] << 8 | w[i];
559 }
560
561 CONTINUE(&x32, &y32, &z32, &w32);
562 }
563 #endif /* BIT_DEPTH == 8 */
564
565 29760 SWS_FOR(PX, EXPAND_BIT, DECL_IMPL, expand_bit)
566 136064 SWS_FOR(PX, EXPAND_PAIR, DECL_IMPL, expand_pair)
567 SWS_FOR(PX, EXPAND_QUAD, DECL_IMPL, expand_quad)
568 SWS_FOR_STRUCT(PX, EXPAND_BIT, DECL_ENTRY)
569 SWS_FOR_STRUCT(PX, EXPAND_PAIR, DECL_ENTRY)
570 SWS_FOR_STRUCT(PX, EXPAND_QUAD, DECL_ENTRY)
571
572 /*************************
573 * Packing and unpacking *
574 ************************/
575
576 #if !IS_FLOAT
577 7794880 DECL_FUNC(unpack, const SwsCompMask mask,
578 const uint8_t bx, const uint8_t by,
579 const uint8_t bz, const uint8_t bw)
580 {
581 7794880 const uint8_t sx = bw + bz + by;
582 7794880 const uint8_t sy = bw + bz;
583 7794880 const uint8_t sz = bw;
584 7794880 const uint8_t sw = 0;
585
586 7794880 const pixel_t mx = (1 << bx) - 1;
587 7794880 const pixel_t my = (1 << by) - 1;
588 7794880 const pixel_t mz = (1 << bz) - 1;
589 7794880 const pixel_t mw = (1 << bw) - 1;
590
591 SWS_LOOP
592
2/2
✓ Branch 0 taken 124718080 times.
✓ Branch 1 taken 3897440 times.
 257231040 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
593 249436160 const pixel_t val = x[i];
594
2/2
✓ Branch 0 taken 91620352 times.
✓ Branch 1 taken 33097728 times.
 249436160 if (X) x[i] = (val >> sx) & mx;
595
2/2
✓ Branch 0 taken 123451392 times.
✓ Branch 1 taken 1266688 times.
 249436160 if (Y) y[i] = (val >> sy) & my;
596
2/2
✓ Branch 0 taken 123396096 times.
✓ Branch 1 taken 1321984 times.
 249436160 if (Z) z[i] = (val >> sz) & mz;
597
2/2
✓ Branch 0 taken 30509056 times.
✓ Branch 1 taken 94209024 times.
 249436160 if (W) w[i] = (val >> sw) & mw;
598 }
599
600 7794880 CONTINUE(x, y, z, w);
601 7794880 }
602
603 7590208 DECL_FUNC(pack, const SwsCompMask mask,
604 const uint8_t bx, const uint8_t by,
605 const uint8_t bz, const uint8_t bw)
606 {
607 7590208 const uint8_t sx = bw + bz + by;
608 7590208 const uint8_t sy = bw + bz;
609 7590208 const uint8_t sz = bw;
610 7590208 const uint8_t sw = 0;
611
612 SWS_LOOP
613
2/2
✓ Branch 0 taken 121443328 times.
✓ Branch 1 taken 3795104 times.
 250476864 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
614 242886656 pixel_t val = 0;
615
1/2
✓ Branch 0 taken 121443328 times.
✗ Branch 1 not taken.
 242886656 if (X) val |= x[i] << sx;
616
1/2
✓ Branch 0 taken 121443328 times.
✗ Branch 1 not taken.
 242886656 if (Y) val |= y[i] << sy;
617
1/2
✓ Branch 0 taken 121443328 times.
✗ Branch 1 not taken.
 242886656 if (Z) val |= z[i] << sz;
618
2/2
✓ Branch 0 taken 37726208 times.
✓ Branch 1 taken 83717120 times.
 242886656 if (W) val |= w[i] << sw;
619 242886656 x[i] = val;
620 }
621
622 7590208 CONTINUE(x, y, z, w);
623 7590208 }
624 #endif /* !IS_FLOAT */
625
626 7794880 SWS_FOR(PX, UNPACK, DECL_IMPL, unpack)
627 7590208 SWS_FOR(PX, PACK, DECL_IMPL, pack)
628 SWS_FOR_STRUCT(PX, UNPACK, DECL_ENTRY)
629 SWS_FOR_STRUCT(PX, PACK, DECL_ENTRY)
630
631 /***********************
632 * Pixel data clearing *
633 ***********************/
634
635 #ifdef PIXEL_MAX
636 13424768 DECL_FUNC(clear, const SwsCompMask mask, const SwsCompMask one,
637 const SwsCompMask zero)
638 {
639 #define ONE(N) SWS_COMP_TEST(one, N)
640 #define ZERO(N) SWS_COMP_TEST(zero, N)
641
4/4
✓ Branch 0 taken 5574976 times.
✓ Branch 1 taken 1137408 times.
✓ Branch 2 taken 5278496 times.
✓ Branch 3 taken 296480 times.
 13424768 const pixel_t cx = ONE(0) ? PIXEL_MAX : ZERO(0) ? 0 : impl->priv.px[0];
642
3/4
✓ Branch 0 taken 6392384 times.
✓ Branch 1 taken 320000 times.
✓ Branch 2 taken 6392384 times.
✗ Branch 3 not taken.
 13424768 const pixel_t cy = ONE(1) ? PIXEL_MAX : ZERO(1) ? 0 : impl->priv.px[1];
643
2/4
✓ Branch 0 taken 6712384 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6712384 times.
✗ Branch 3 not taken.
 13424768 const pixel_t cz = ONE(2) ? PIXEL_MAX : ZERO(2) ? 0 : impl->priv.px[2];
644
4/4
✓ Branch 0 taken 5908064 times.
✓ Branch 1 taken 804320 times.
✓ Branch 2 taken 4867520 times.
✓ Branch 3 taken 1040544 times.
 13424768 const pixel_t cw = ONE(3) ? PIXEL_MAX : ZERO(3) ? 0 : impl->priv.px[3];
645
646 SWS_LOOP
647
2/2
✓ Branch 0 taken 214796288 times.
✓ Branch 1 taken 6712384 times.
 443017344 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
648
2/2
✓ Branch 0 taken 127380480 times.
✓ Branch 1 taken 87415808 times.
 429592576 if (X) x[i] = cx;
649
2/2
✓ Branch 0 taken 37572608 times.
✓ Branch 1 taken 177223680 times.
 429592576 if (Y) y[i] = cy;
650
2/2
✓ Branch 0 taken 12679168 times.
✓ Branch 1 taken 202117120 times.
 429592576 if (Z) z[i] = cz;
651
2/2
✓ Branch 0 taken 79071232 times.
✓ Branch 1 taken 135725056 times.
 429592576 if (W) w[i] = cw;
652 }
653
654 13424768 CONTINUE(x, y, z, w);
655 13424768 }
656 #endif
657
658 13424768 SWS_FOR(PX, CLEAR, DECL_IMPL, clear)
659 SWS_FOR_STRUCT(PX, CLEAR, DECL_ENTRY, .setup = ff_sws_setup_vec4)
660
661 /*************************
662 * Arithmetic operations *
663 *************************/
664
665 11906048 DECL_FUNC(scale, const SwsCompMask mask)
666 {
667 11906048 const pixel_t scale = impl->priv.px[0];
668
669 SWS_LOOP
670
2/2
✓ Branch 0 taken 190496768 times.
✓ Branch 1 taken 5953024 times.
 392899584 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
671
2/2
✓ Branch 0 taken 184336384 times.
✓ Branch 1 taken 6160384 times.
 380993536 if (X) x[i] *= scale;
672
2/2
✓ Branch 0 taken 148362240 times.
✓ Branch 1 taken 42134528 times.
 380993536 if (Y) y[i] *= scale;
673
2/2
✓ Branch 0 taken 144888832 times.
✓ Branch 1 taken 45607936 times.
 380993536 if (Z) z[i] *= scale;
674
2/2
✓ Branch 0 taken 18852864 times.
✓ Branch 1 taken 171643904 times.
 380993536 if (W) w[i] *= scale;
675 }
676
677 11906048 CONTINUE(x, y, z, w);
678 11906048 }
679
680 654880 DECL_FUNC(add, const SwsCompMask mask)
681 {
682 SWS_LOOP
683
2/2
✓ Branch 0 taken 20956160 times.
✓ Branch 1 taken 654880 times.
 21611040 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
684
2/2
✓ Branch 0 taken 20837376 times.
✓ Branch 1 taken 118784 times.
 20956160 if (X) x[i] += impl->priv.px[0];
685
2/2
✓ Branch 0 taken 14868480 times.
✓ Branch 1 taken 6087680 times.
 20956160 if (Y) y[i] += impl->priv.px[1];
686
2/2
✓ Branch 0 taken 14726144 times.
✓ Branch 1 taken 6230016 times.
 20956160 if (Z) z[i] += impl->priv.px[2];
687
2/2
✓ Branch 0 taken 2522112 times.
✓ Branch 1 taken 18434048 times.
 20956160 if (W) w[i] += impl->priv.px[3];
688 }
689
690 654880 CONTINUE(x, y, z, w);
691 654880 }
692
693 13604448 DECL_FUNC(min, const SwsCompMask mask)
694 {
695 SWS_LOOP
696
2/2
✓ Branch 0 taken 435342336 times.
✓ Branch 1 taken 13604448 times.
 448946784 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
697
2/2
✓ Branch 0 taken 425750528 times.
✓ Branch 1 taken 9591808 times.
 435342336 if (X) x[i] = FFMIN(x[i], impl->priv.px[0]);
698
2/2
✓ Branch 0 taken 331380736 times.
✓ Branch 1 taken 103961600 times.
 435342336 if (Y) y[i] = FFMIN(y[i], impl->priv.px[1]);
699
2/2
✓ Branch 0 taken 326767616 times.
✓ Branch 1 taken 108574720 times.
 435342336 if (Z) z[i] = FFMIN(z[i], impl->priv.px[2]);
700
2/2
✓ Branch 0 taken 39075840 times.
✓ Branch 1 taken 396266496 times.
 435342336 if (W) w[i] = FFMIN(w[i], impl->priv.px[3]);
701 }
702
703 13604448 CONTINUE(x, y, z, w);
704 13604448 }
705
706 10324768 DECL_FUNC(max, const SwsCompMask mask)
707 {
708 SWS_LOOP
709
2/2
✓ Branch 0 taken 330392576 times.
✓ Branch 1 taken 10324768 times.
 340717344 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
710
2/2
✓ Branch 0 taken 330388480 times.
✓ Branch 1 taken 4096 times.
 330392576 if (X) x[i] = FFMAX(x[i], impl->priv.px[0]);
711
2/2
✓ Branch 0 taken 253825024 times.
✓ Branch 1 taken 76567552 times.
 330392576 if (Y) y[i] = FFMAX(y[i], impl->priv.px[1]);
712
2/2
✓ Branch 0 taken 251681792 times.
✓ Branch 1 taken 78710784 times.
 330392576 if (Z) z[i] = FFMAX(z[i], impl->priv.px[2]);
713
2/2
✓ Branch 0 taken 28923904 times.
✓ Branch 1 taken 301468672 times.
 330392576 if (W) w[i] = FFMAX(w[i], impl->priv.px[3]);
714 }
715
716 10324768 CONTINUE(x, y, z, w);
717 10324768 }
718
719 11906048 SWS_FOR(PX, SCALE, DECL_IMPL, scale)
720 1309760 SWS_FOR(PX, ADD, DECL_IMPL, add)
721 27208896 SWS_FOR(PX, MIN, DECL_IMPL, min)
722 20649536 SWS_FOR(PX, MAX, DECL_IMPL, max)
723 SWS_FOR_STRUCT(PX, SCALE, DECL_ENTRY, .setup = ff_sws_setup_scalar )
724 SWS_FOR_STRUCT(PX, ADD, DECL_ENTRY, .setup = ff_sws_setup_vec4 )
725 SWS_FOR_STRUCT(PX, MIN, DECL_ENTRY, .setup = ff_sws_setup_vec4 )
726 SWS_FOR_STRUCT(PX, MAX, DECL_ENTRY, .setup = ff_sws_setup_vec4 )
727
728 /*************
729 * Dithering *
730 *************/
731
732 72574 DECL_SETUP(setup_dither, params, out)
733 {
734 72574 const SwsUOp *uop = params->uop;
735 72574 const SwsDitherUOp *dither = &uop->par.dither;
736 72574 const int size = 1 << dither->size_log2;
737
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36287 times.
 72574 if (size >= SWS_BLOCK_SIZE) {
738 /* No extra padding needed */
739 ✗ out->priv.ptr = av_refstruct_ref(uop->data.ptr);
740 ✗ out->free = ff_op_priv_unref;
741 ✗ return 0;
742 }
743
744 72574 const int stride = FFMAX(size, SWS_BLOCK_SIZE);
745 72574 const int height = ff_sws_dither_height(dither);
746 72574 pixel_t *matrix = av_malloc(sizeof(pixel_t) * height * stride);
747
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 36287 times.
 72574 if (!matrix)
748 ✗ return AVERROR(ENOMEM);
749 72574 out->priv.ptr = matrix;
750 72574 out->free = ff_op_priv_free;
751
752 /* Pad to multiple of block size. We don't need extra padding for the
753 * height because ff_sws_dither_height() already includes any padding
754 * necessary for the y_offset */
755
2/2
✓ Branch 0 taken 675414 times.
✓ Branch 1 taken 36287 times.
 1423402 for (int y = 0; y < height; y++) {
756 1350828 pixel_t *row = &matrix[y * stride];
757
2/2
✓ Branch 0 taken 10806624 times.
✓ Branch 1 taken 675414 times.
 22964076 for (int x = 0; x < size; x++)
758 21613248 row[x] = uop->data.ptr[y * size + x].px;
759
2/2
✓ Branch 0 taken 10806624 times.
✓ Branch 1 taken 675414 times.
 22964076 for (int x = size; x < stride; x++)
760 21613248 row[x] = row[x % size];
761 }
762
763 72574 return 0;
764 }
765
766 10358656 DECL_FUNC(dither, const SwsCompMask mask,
767 const uint8_t off0, const uint8_t off1,
768 const uint8_t off2, const uint8_t off3,
769 const uint8_t size_log2)
770 {
771 10358656 const int size = 1 << size_log2;
772 10358656 const int stride = FFMAX(size, SWS_BLOCK_SIZE);
773
774 10358656 const pixel_t *matrix = impl->priv.ptr;
775 10358656 matrix += (iter->y & (size - 1)) * stride;
776 10358656 matrix += (iter->x & (size - 1)) & ~(SWS_BLOCK_SIZE - 1);
777
778 10358656 const pixel_t *const row0 = &matrix[off0 * stride];
779 10358656 const pixel_t *const row1 = &matrix[off1 * stride];
780 10358656 const pixel_t *const row2 = &matrix[off2 * stride];
781 10358656 const pixel_t *const row3 = &matrix[off3 * stride];
782
783 SWS_LOOP
784
2/2
✓ Branch 0 taken 331476992 times.
✓ Branch 1 taken 10358656 times.
 341835648 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
785
2/2
✓ Branch 0 taken 324962304 times.
✓ Branch 1 taken 6514688 times.
 331476992 if (X) x[i] += row0[i];
786
2/2
✓ Branch 0 taken 263610368 times.
✓ Branch 1 taken 67866624 times.
 331476992 if (Y) y[i] += row1[i];
787
2/2
✓ Branch 0 taken 260293632 times.
✓ Branch 1 taken 71183360 times.
 331476992 if (Z) z[i] += row2[i];
788
2/2
✓ Branch 0 taken 35826688 times.
✓ Branch 1 taken 295650304 times.
 331476992 if (W) w[i] += row3[i];
789 }
790
791 10358656 CONTINUE(x, y, z, w);
792 10358656 }
793
794 20717312 SWS_FOR(PX, DITHER, DECL_IMPL, dither)
795 SWS_FOR_STRUCT(PX, DITHER, DECL_ENTRY, .setup = fn(setup_dither) )
796
797 /*********************
798 * Linear operations *
799 *********************/
800
801 typedef struct {
802 /* Stored in split form for convenience */
803 pixel_t m[4][4];
804 pixel_t k[4];
805 } fn(LinCoeffs);
806
807 85518 DECL_SETUP(setup_linear, params, out)
808 {
809 85518 const SwsUOp *uop = params->uop;
810 fn(LinCoeffs) c;
811
812
2/2
✓ Branch 0 taken 171036 times.
✓ Branch 1 taken 42759 times.
 427590 for (int i = 0; i < 4; i++) {
813
2/2
✓ Branch 0 taken 684144 times.
✓ Branch 1 taken 171036 times.
 1710360 for (int j = 0; j < 4; j++)
814 1368288 c.m[i][j] = uop->data.mat4x5[i][j].px;
815 342072 c.k[i] = uop->data.mat4x5[i][4].px;
816 }
817
818 85518 out->priv.ptr = av_memdup(&c, sizeof(c));
819 85518 out->free = ff_op_priv_free;
820
1/2
✓ Branch 0 taken 42759 times.
✗ Branch 1 not taken.
 85518 return out->priv.ptr ? 0 : AVERROR(ENOMEM);
821 }
822
823 /**
824 * Fully general case for a 5x5 linear affine transformation. Should never be
825 * called without constant `mask`. This function will compile down to the
826 * appropriately optimized version for the required subset of operations when
827 * called with a constant mask.
828 */
829 24461504 DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zero)
830 {
831 24461504 const fn(LinCoeffs) c = *(const fn(LinCoeffs) *) impl->priv.ptr;
832
833 SWS_LOOP
834
2/2
✓ Branch 0 taken 391384064 times.
✓ Branch 1 taken 12230752 times.
 807229632 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
835 782768128 const pixel_t xx = x[i];
836 782768128 const pixel_t yy = y[i];
837 782768128 const pixel_t zz = z[i];
838 782768128 const pixel_t ww = w[i];
839
840 #define LIN_VAL(I, J, val) \
841 ((one & SWS_MASK(I, J)) ? (val) : (uinter_t) c.m[I][J] * (val))
842
843 #define LIN_ROW(I, var) do { \
844 pixel_t tmp = (zero & SWS_MASK(I, 4)) ? 0 : c.k[I]; \
845 if (!(zero & SWS_MASK(I, 0))) tmp += LIN_VAL(I, 0, xx); \
846 if (!(zero & SWS_MASK(I, 1))) tmp += LIN_VAL(I, 1, yy); \
847 if (!(zero & SWS_MASK(I, 2))) tmp += LIN_VAL(I, 2, zz); \
848 if (!(zero & SWS_MASK(I, 3))) tmp += LIN_VAL(I, 3, ww); \
849 var[i] = tmp; \
850 } while (0)
851
852
14/20
✓ Branch 0 taken 386535424 times.
✓ Branch 1 taken 4848640 times.
✓ Branch 2 taken 274654208 times.
✓ Branch 3 taken 111881216 times.
✓ Branch 4 taken 386535424 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 386199552 times.
✓ Branch 7 taken 335872 times.
✓ Branch 8 taken 158984192 times.
✓ Branch 9 taken 227551232 times.
✓ Branch 10 taken 158984192 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 262398976 times.
✓ Branch 13 taken 124136448 times.
✓ Branch 14 taken 262398976 times.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 386535424 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 782768128 if (X) LIN_ROW(0, x);
853
14/20
✓ Branch 0 taken 281206784 times.
✓ Branch 1 taken 110177280 times.
✓ Branch 2 taken 216443904 times.
✓ Branch 3 taken 64762880 times.
✓ Branch 4 taken 211350528 times.
✓ Branch 5 taken 69856256 times.
✓ Branch 6 taken 211014656 times.
✓ Branch 7 taken 335872 times.
✓ Branch 8 taken 281206784 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 281206784 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 211350528 times.
✓ Branch 13 taken 69856256 times.
✓ Branch 14 taken 211350528 times.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 281206784 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 782768128 if (Y) LIN_ROW(1, y);
854
15/20
✓ Branch 0 taken 276882432 times.
✓ Branch 1 taken 114501632 times.
✓ Branch 2 taken 216443904 times.
✓ Branch 3 taken 60438528 times.
✓ Branch 4 taken 211350528 times.
✓ Branch 5 taken 65531904 times.
✓ Branch 6 taken 211014656 times.
✓ Branch 7 taken 335872 times.
✓ Branch 8 taken 211350528 times.
✓ Branch 9 taken 65531904 times.
✓ Branch 10 taken 211350528 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 173467648 times.
✓ Branch 13 taken 103414784 times.
✓ Branch 14 taken 173467648 times.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 276882432 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 782768128 if (Z) LIN_ROW(2, z);
855
8/20
✓ Branch 0 taken 42160128 times.
✓ Branch 1 taken 349223936 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 42160128 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 42160128 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 42160128 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 42160128 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 42160128 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 42160128 times.
✗ Branch 19 not taken.
 782768128 if (W) LIN_ROW(3, w);
856 }
857
858 24461504 CONTINUE(x, y, z, w);
859 24461504 }
860
861 24461504 SWS_FOR(PX, LINEAR, DECL_IMPL, linear)
862 SWS_FOR_STRUCT(PX, LINEAR, DECL_ENTRY, .setup = fn(setup_linear) )
863
864 /******************
865 * Look-up tables *
866 ******************/
867
868 112 DECL_SETUP(setup_lut3d, params, out)
869 {
870 112 const SwsLut3D *lut = params->uop->data.lut3d;
871 112 out->priv.ptr = (void *) av_refstruct_ref_c(lut);
872 112 out->free = ff_op_priv_unref;
873 112 return 0;
874 }
875
876 #if IS_FLOAT
877 98304 av_always_inline static vec3_t fn(vec3)(v3u16_t v)
878 {
879 98304 return (vec3_t) { v.x, v.y, v.z };
880 }
881
882 #define lerp(a, b, w) ((a) + (w) * ((pixel_t) (b) - (a)))
883
884 av_always_inline static
885 43008 vec3_t fn(lerp3)(vec3_t a, vec3_t b, pixel_t w)
886 {
887 86016 return (vec3_t) {
888 43008 lerp(a.x, b.x, w),
889 43008 lerp(a.y, b.y, w),
890 43008 lerp(a.z, b.z, w),
891 };
892 }
893
894 av_always_inline static
895 12288 vec3_t fn(lut3d_static)(const SwsLut3D *restrict lut3d, vec3_t rgb)
896 {
897 12288 const int r_base = (int) rgb.x;
898 12288 const int g_base = (int) rgb.y;
899 12288 const int b_base = (int) rgb.z;
900
901 12288 int off0 = (r_base < INPUT_LUT_SIZE - 1);
902
1/2
✓ Branch 0 taken 12288 times.
✗ Branch 1 not taken.
 12288 int off1 = (g_base < INPUT_LUT_SIZE - 1) * INPUT_LUT_SIZE;
903
1/2
✓ Branch 0 taken 12288 times.
✗ Branch 1 not taken.
 12288 int off2 = (b_base < INPUT_LUT_SIZE - 1) * INPUT_LUT_SIZE * INPUT_LUT_SIZE;
904 12288 pixel_t f0 = rgb.x - r_base;
905 12288 pixel_t f1 = rgb.y - g_base;
906 12288 pixel_t f2 = rgb.z - b_base;
907
908 /* Sort offsets descending by relative weight */
909
2/2
✓ Branch 0 taken 5880 times.
✓ Branch 1 taken 6408 times.
 12288 if (f0 < f1) {
910 5880 FFSWAP(pixel_t, f0, f1);
911 5880 FFSWAP(int, off0, off1);
912 }
913
2/2
✓ Branch 0 taken 4188 times.
✓ Branch 1 taken 8100 times.
 12288 if (f0 < f2) {
914 4188 FFSWAP(pixel_t, f0, f2);
915 4188 FFSWAP(int, off0, off2);
916 }
917
2/2
✓ Branch 0 taken 8004 times.
✓ Branch 1 taken 4284 times.
 12288 if (f1 < f2) {
918 8004 FFSWAP(pixel_t, f1, f2);
919 8004 FFSWAP(int, off1, off2);
920 }
921
922 /* Tetrahedral interpolation */
923 12288 const pixel_t w0 = 1 - f0;
924 12288 const pixel_t w1 = f0 - f1;
925 12288 const pixel_t w2 = f1 - f2;
926 12288 const pixel_t w3 = f2;
927
928 12288 const v3u16_t *restrict base = &lut3d->input[b_base][g_base][r_base];
929 12288 const vec3_t v0 = fn(vec3)(base[0]);
930 12288 const vec3_t v1 = fn(vec3)(base[off0]);
931 12288 const vec3_t v2 = fn(vec3)(base[off0 + off1]);
932 12288 const vec3_t v3 = fn(vec3)(base[off0 + off1 + off2]);
933
934 24576 return (vec3_t) {
935 12288 w0 * v0.x + w1 * v1.x + w2 * v2.x + w3 * v3.x,
936 12288 w0 * v0.y + w1 * v1.y + w2 * v2.y + w3 * v3.y,
937 12288 w0 * v0.z + w1 * v1.z + w2 * v2.z + w3 * v3.z,
938 };
939 }
940
941 av_always_inline static
942 6144 vec3_t fn(lut3d_dynamic)(const SwsLut3D *restrict lut3d, vec3_t rgb)
943 {
944 6144 rgb.x *= (TONE_LUT_SIZE - 1) / (pixel_t) UINT16_MAX;
945
946 /* Linear interpolation */
947 6144 const int Ix = (int) rgb.x;
948 6144 const pixel_t If = rgb.x - Ix;
949
950 6144 const v2u16_t a = lut3d->tone_map[Ix];
951 6144 const v2u16_t b = lut3d->tone_map[Ix + 1];
952
953 6144 const pixel_t k = lerp(a.y, b.y, If);
954 6144 const pixel_t bias = (1 << 15) - k;
955 6144 const pixel_t scale = k / (pixel_t) (1 << 15);
956
957 6144 rgb.x = lerp(a.x, b.x, If);
958 6144 rgb.y = bias + scale * rgb.y;
959 6144 rgb.z = bias + scale * rgb.z;
960
961 /* Re-scale to output LUT size */
962 6144 rgb.x *= (OUTPUT_LUT_SIZE_I - 1) / (pixel_t) UINT16_MAX;
963 6144 rgb.y *= (OUTPUT_LUT_SIZE_PT - 1) / (pixel_t) UINT16_MAX;
964 6144 rgb.z *= (OUTPUT_LUT_SIZE_PT - 1) / (pixel_t) UINT16_MAX;
965
966 /* Trilinear interpolation */
967 6144 const int lo0 = (int) rgb.x;
968 6144 const int lo1 = (int) rgb.y;
969 6144 const int lo2 = (int) rgb.z;
970
971 6144 const int hi0 = FFMIN(lo0 + 1, OUTPUT_LUT_SIZE_I - 1);
972 6144 const int hi1 = FFMIN(lo1 + 1, OUTPUT_LUT_SIZE_PT - 1);
973 6144 const int hi2 = FFMIN(lo2 + 1, OUTPUT_LUT_SIZE_PT - 1);
974
975 6144 const pixel_t w0 = rgb.x - lo0;
976 6144 const vec3_t c000 = fn(vec3)(lut3d->output[lo2][lo1][lo0]);
977 6144 const vec3_t c001 = fn(vec3)(lut3d->output[lo2][lo1][hi0]);
978 6144 const vec3_t c00 = fn(lerp3)(c000, c001, w0);
979 6144 const vec3_t c010 = fn(vec3)(lut3d->output[lo2][hi1][lo0]);
980 6144 const vec3_t c011 = fn(vec3)(lut3d->output[lo2][hi1][hi0]);
981 6144 const vec3_t c01 = fn(lerp3)(c010, c011, w0);
982 6144 const vec3_t c100 = fn(vec3)(lut3d->output[hi2][lo1][lo0]);
983 6144 const vec3_t c101 = fn(vec3)(lut3d->output[hi2][lo1][hi0]);
984 6144 const vec3_t c10 = fn(lerp3)(c100, c101, w0);
985 6144 const vec3_t c110 = fn(vec3)(lut3d->output[hi2][hi1][lo0]);
986 6144 const vec3_t c111 = fn(vec3)(lut3d->output[hi2][hi1][hi0]);
987 6144 const vec3_t c11 = fn(lerp3)(c110, c111, w0);
988
989 6144 const pixel_t w1 = rgb.y - lo1;
990 6144 const vec3_t c0 = fn(lerp3)(c00, c01, w1);
991 6144 const vec3_t c1 = fn(lerp3)(c10, c11, w1);
992
993 6144 const pixel_t w2 = rgb.z - lo2;
994 6144 return fn(lerp3)(c0, c1, w2);
995 }
996
997 384 DECL_FUNC(lut3d, const SwsCompMask mask, const int dynamic)
998 {
999 384 const SwsLut3D *restrict lut3d = impl->priv.ptr;
1000
1001 SWS_LOOP
1002
2/2
✓ Branch 0 taken 12288 times.
✓ Branch 1 taken 384 times.
 12672 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
1003 12288 vec3_t c = { x[i], y[i], z[i] };
1004 12288 c = fn(lut3d_static)(lut3d, c);
1005
2/2
✓ Branch 0 taken 6144 times.
✓ Branch 1 taken 6144 times.
 12288 if (dynamic)
1006 6144 c = fn(lut3d_dynamic)(lut3d, c);
1007
1008 12288 x[i] = c.x;
1009 12288 y[i] = c.y;
1010 12288 z[i] = c.z;
1011 }
1012
1013 384 CONTINUE(x, y, z, w);
1014 384 }
1015 #endif /* IS_FLOAT */
1016
1017 768 SWS_FOR(PX, LUT_3D, DECL_IMPL, lut3d)
1018 SWS_FOR_STRUCT(PX, LUT_3D, DECL_ENTRY, .setup = fn(setup_lut3d) )
1019
1020 #undef PIXEL_MAX
1021 #undef PIXEL_SWAP
1022 #undef pixel_t
1023 #undef inter_t
1024 #undef uinter_t
1025 #undef vec3_t
1026 #undef PX
1027 #undef px
1028