FFmpeg coverage

Directory: ../../../ffmpeg/
File: src/libswscale/uops_tmpl.c
Date: 2026-09-17 19:38:28
Exec Total Coverage
Lines: 489 503 97.2%
Functions: 456 467 97.6%
Branches: 391 464 84.3%
Line Branch Exec Source
1 /**
2 * Copyright (C) 2026 Niklas Haas
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #include <libavutil/bswap.h>
22
23 #include "uops_tmpl.h"
24
25 #ifndef BIT_DEPTH
26 # define BIT_DEPTH 8
27 #endif
28
29 #if IS_FLOAT && BIT_DEPTH == 32
30 # define PIXEL_TYPE SWS_PIXEL_F32
31 # define pixel_t float
32 # define inter_t float
33 # define vec3_t v3f32_t
34 # define PX F32
35 # define px f32
36 #elif BIT_DEPTH == 32
37 # define PIXEL_MAX 0xFFFFFFFFu
38 # define PIXEL_SWAP av_bswap32
39 # define pixel_t uint32_t
40 # define inter_t int64_t
41 # define PX U32
42 # define px u32
43 #elif BIT_DEPTH == 16
44 # define PIXEL_MAX 0xFFFFu
45 # define PIXEL_SWAP av_bswap16
46 # define pixel_t uint16_t
47 # define inter_t int64_t
48 # define PX U16
49 # define px u16
50 #elif BIT_DEPTH == 8
51 # define PIXEL_MAX 0xFFu
52 # define pixel_t uint8_t
53 # define inter_t int32_t
54 # define PX U8
55 # define px u8
56 #else
57 # error Invalid BIT_DEPTH
58 #endif
59
60 /*********************************
61 * Generic read/write operations *
62 *********************************/
63
64 1532608 DECL_READ(read_planar, const SwsCompMask mask)
65 {
66 SWS_LOOP
67
2/2
✓ Branch 0 taken 24521728 times.
✓ Branch 1 taken 766304 times.
 50576064 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
68
1/2
✓ Branch 0 taken 24521728 times.
✗ Branch 1 not taken.
 49043456 if (X) x[i] = in0[i];
69
2/2
✓ Branch 0 taken 6774784 times.
✓ Branch 1 taken 17746944 times.
 49043456 if (Y) y[i] = in1[i];
70
2/2
✓ Branch 0 taken 6764544 times.
✓ Branch 1 taken 17757184 times.
 49043456 if (Z) z[i] = in2[i];
71
2/2
✓ Branch 0 taken 2970624 times.
✓ Branch 1 taken 21551104 times.
 49043456 if (W) w[i] = in3[i];
72 }
73
74
1/2
✓ Branch 0 taken 766304 times.
✗ Branch 1 not taken.
 1532608 if (X) iter->in[0] += SIZEOF_BLOCK;
75
2/2
✓ Branch 0 taken 211712 times.
✓ Branch 1 taken 554592 times.
 1532608 if (Y) iter->in[1] += SIZEOF_BLOCK;
76
2/2
✓ Branch 0 taken 211392 times.
✓ Branch 1 taken 554912 times.
 1532608 if (Z) iter->in[2] += SIZEOF_BLOCK;
77
2/2
✓ Branch 0 taken 92832 times.
✓ Branch 1 taken 673472 times.
 1532608 if (W) iter->in[3] += SIZEOF_BLOCK;
78
79 1532608 CONTINUE(x, y, z, w);
80 1532608 }
81
82 240960 DECL_READ(read_packed, const SwsCompMask mask)
83 {
84
5/6
✓ Branch 0 taken 51808 times.
✓ Branch 1 taken 68672 times.
✓ Branch 2 taken 17696 times.
✓ Branch 3 taken 34112 times.
✓ Branch 4 taken 17696 times.
✗ Branch 5 not taken.
 240960 const int elems = W ? 4 : Z ? 3 : Y ? 2 : 1;
85
86 SWS_LOOP
87
2/2
✓ Branch 0 taken 3855360 times.
✓ Branch 1 taken 120480 times.
 7951680 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
88
1/2
✓ Branch 0 taken 3855360 times.
✗ Branch 1 not taken.
 7710720 if (X) x[i] = in0[elems * i + 0];
89
1/2
✓ Branch 0 taken 3855360 times.
✗ Branch 1 not taken.
 7710720 if (Y) y[i] = in0[elems * i + 1];
90
2/2
✓ Branch 0 taken 3289088 times.
✓ Branch 1 taken 566272 times.
 7710720 if (Z) z[i] = in0[elems * i + 2];
91
2/2
✓ Branch 0 taken 2197504 times.
✓ Branch 1 taken 1657856 times.
 7710720 if (W) w[i] = in0[elems * i + 3];
92 }
93
94 240960 iter->in[0] += SIZEOF_BLOCK * elems;
95 240960 CONTINUE(x, y, z, w);
96 240960 }
97
98 3910400 DECL_WRITE(write_planar, const SwsCompMask mask)
99 {
100 SWS_LOOP
101
2/2
✓ Branch 0 taken 62566400 times.
✓ Branch 1 taken 1955200 times.
 129043200 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
102
1/2
✓ Branch 0 taken 62566400 times.
✗ Branch 1 not taken.
 125132800 if (X) out0[i] = x[i];
103
2/2
✓ Branch 0 taken 15744000 times.
✓ Branch 1 taken 46822400 times.
 125132800 if (Y) out1[i] = y[i];
104
2/2
✓ Branch 0 taken 15728640 times.
✓ Branch 1 taken 46837760 times.
 125132800 if (Z) out2[i] = z[i];
105
2/2
✓ Branch 0 taken 4971520 times.
✓ Branch 1 taken 57594880 times.
 125132800 if (W) out3[i] = w[i];
106 }
107
108
1/2
✓ Branch 0 taken 1955200 times.
✗ Branch 1 not taken.
 3910400 if (X) iter->out[0] += SIZEOF_BLOCK;
109
2/2
✓ Branch 0 taken 492000 times.
✓ Branch 1 taken 1463200 times.
 3910400 if (Y) iter->out[1] += SIZEOF_BLOCK;
110
2/2
✓ Branch 0 taken 491520 times.
✓ Branch 1 taken 1463680 times.
 3910400 if (Z) iter->out[2] += SIZEOF_BLOCK;
111
2/2
✓ Branch 0 taken 155360 times.
✓ Branch 1 taken 1799840 times.
 3910400 if (W) iter->out[3] += SIZEOF_BLOCK;
112 3910400 }
113
114 813760 DECL_WRITE(write_packed, const SwsCompMask mask)
115 {
116
5/6
✓ Branch 0 taken 155584 times.
✓ Branch 1 taken 251296 times.
✓ Branch 2 taken 128608 times.
✓ Branch 3 taken 26976 times.
✓ Branch 4 taken 128608 times.
✗ Branch 5 not taken.
 813760 const int elems = W ? 4 : Z ? 3 : Y ? 2 : 1;
117
118 SWS_LOOP
119
2/2
✓ Branch 0 taken 13020160 times.
✓ Branch 1 taken 406880 times.
 26854080 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
120
1/2
✓ Branch 0 taken 13020160 times.
✗ Branch 1 not taken.
 26040320 if (X) out0[elems * i + 0] = x[i];
121
1/2
✓ Branch 0 taken 13020160 times.
✗ Branch 1 not taken.
 26040320 if (Y) out0[elems * i + 1] = y[i];
122
2/2
✓ Branch 0 taken 8904704 times.
✓ Branch 1 taken 4115456 times.
 26040320 if (Z) out0[elems * i + 2] = z[i];
123
2/2
✓ Branch 0 taken 8041472 times.
✓ Branch 1 taken 4978688 times.
 26040320 if (W) out0[elems * i + 3] = w[i];
124 }
125
126 813760 iter->out[0] += SIZEOF_BLOCK * elems;
127 813760 }
128
129 #if BIT_DEPTH == 8
130
131 9952 DECL_READ(read_bit, const SwsCompMask mask)
132 {
133 av_assert2(mask == SWS_COMP_ELEMS(1));
134
135 SWS_LOOP
136
2/2
✓ Branch 0 taken 39808 times.
✓ Branch 1 taken 9952 times.
 49760 for (int i = 0; i < SWS_BLOCK_SIZE; i += 8) {
137 39808 const pixel_t val = ((const pixel_t *) in0)[i >> 3];
138 39808 x[i + 0] = (val >> 7) & 1;
139 39808 x[i + 1] = (val >> 6) & 1;
140 39808 x[i + 2] = (val >> 5) & 1;
141 39808 x[i + 3] = (val >> 4) & 1;
142 39808 x[i + 4] = (val >> 3) & 1;
143 39808 x[i + 5] = (val >> 2) & 1;
144 39808 x[i + 6] = (val >> 1) & 1;
145 39808 x[i + 7] = (val >> 0) & 1;
146 }
147
148 9952 iter->in[0] += SIZEOF_BLOCK >> 3;
149 9952 CONTINUE(x, y, z, w);
150 9952 }
151
152 7648 DECL_READ(read_nibble, const SwsCompMask mask)
153 {
154 av_assert2(mask == SWS_COMP_ELEMS(1));
155
156 SWS_LOOP
157
2/2
✓ Branch 0 taken 122368 times.
✓ Branch 1 taken 7648 times.
 130016 for (int i = 0; i < SWS_BLOCK_SIZE; i += 2) {
158 122368 const pixel_t val = in0[i >> 1];
159 122368 x[i + 0] = val >> 4; /* high nibble */
160 122368 x[i + 1] = val & 0xF; /* low nibble */
161 }
162
163 7648 iter->in[0] += SIZEOF_BLOCK >> 1;
164 7648 CONTINUE(x, y, z, w);
165 7648 }
166
167 160 DECL_READ(read_palette, const SwsCompMask mask)
168 {
169 av_assert2(mask == SWS_COMP_ELEMS(4));
170
171 SWS_LOOP
172
2/2
✓ Branch 0 taken 5120 times.
✓ Branch 1 taken 160 times.
 5280 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
173 5120 const pixel_t index = in0[i];
174 5120 const pixel_t *value = &in1[index * 4];
175 5120 x[i] = value[0];
176 5120 y[i] = value[1];
177 5120 z[i] = value[2];
178 5120 w[i] = value[3];
179 }
180
181 160 iter->in[0] += SIZEOF_BLOCK;
182 160 CONTINUE(x, y, z, w);
183 160 }
184
185 7584 DECL_WRITE(write_bit, const SwsCompMask mask)
186 {
187 av_assert2(mask == SWS_COMP_ELEMS(1));
188
189 SWS_LOOP
190
2/2
✓ Branch 0 taken 30336 times.
✓ Branch 1 taken 7584 times.
 37920 for (int i = 0; i < SWS_BLOCK_SIZE; i += 8) {
191 30336 out0[i >> 3] = x[i + 0] << 7 |
192 30336 x[i + 1] << 6 |
193 30336 x[i + 2] << 5 |
194 30336 x[i + 3] << 4 |
195 30336 x[i + 4] << 3 |
196 30336 x[i + 5] << 2 |
197 30336 x[i + 6] << 1 |
198 30336 x[i + 7];
199 }
200
201 7584 iter->out[0] += SIZEOF_BLOCK >> 3;
202 7584 }
203
204 5088 DECL_WRITE(write_nibble, const SwsCompMask mask)
205 {
206 av_assert2(mask == SWS_COMP_ELEMS(1));
207
208 SWS_LOOP
209
2/2
✓ Branch 0 taken 81408 times.
✓ Branch 1 taken 5088 times.
 86496 for (int i = 0; i < SWS_BLOCK_SIZE; i += 2)
210 81408 out0[i >> 1] = x[i] << 4 | x[i + 1];
211
212 5088 iter->out[0] += SIZEOF_BLOCK >> 1;
213 5088 }
214
215 #endif /* BIT_DEPTH == 8 */
216
217 1532608 SWS_FOR(PX, READ_PLANAR, DECL_IMPL_READ, read_planar)
218 240960 SWS_FOR(PX, READ_PACKED, DECL_IMPL_READ, read_packed)
219 7648 SWS_FOR(PX, READ_NIBBLE, DECL_IMPL_READ, read_nibble)
220 9952 SWS_FOR(PX, READ_BIT, DECL_IMPL_READ, read_bit)
221 160 SWS_FOR(PX, READ_PALETTE, DECL_IMPL_READ, read_palette)
222 3910400 SWS_FOR(PX, WRITE_PLANAR, DECL_IMPL_WRITE, write_planar)
223 813760 SWS_FOR(PX, WRITE_PACKED, DECL_IMPL_WRITE, write_packed)
224 5088 SWS_FOR(PX, WRITE_NIBBLE, DECL_IMPL_WRITE, write_nibble)
225 7584 SWS_FOR(PX, WRITE_BIT, DECL_IMPL_WRITE, write_bit)
226
227 SWS_FOR_STRUCT(PX, READ_PLANAR, DECL_ENTRY)
228 SWS_FOR_STRUCT(PX, READ_PACKED, DECL_ENTRY)
229 SWS_FOR_STRUCT(PX, READ_NIBBLE, DECL_ENTRY)
230 SWS_FOR_STRUCT(PX, READ_BIT, DECL_ENTRY)
231 SWS_FOR_STRUCT(PX, READ_PALETTE, DECL_ENTRY)
232 SWS_FOR_STRUCT(PX, WRITE_PLANAR, DECL_ENTRY)
233 SWS_FOR_STRUCT(PX, WRITE_PACKED, DECL_ENTRY)
234 SWS_FOR_STRUCT(PX, WRITE_NIBBLE, DECL_ENTRY)
235 SWS_FOR_STRUCT(PX, WRITE_BIT, DECL_ENTRY)
236
237 /*****************************
238 * Scaling / filtering reads *
239 *****************************/
240
241 6922 DECL_SETUP(setup_filter_v, params, out)
242 {
243
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3461 times.
 6922 if (params->uop->par.filter.type != SWS_PIXEL_F32)
244 ✗ return AVERROR(ENOTSUP);
245
246 6922 const SwsFilterWeights *filter = params->uop->data.kernel;
247 static_assert(sizeof(out->priv.ptr) <= sizeof(int32_t[2]),
248 ">8 byte pointers not supported");
249
250 /* Pre-convert weights to float */
251 6922 float *weights = av_calloc(filter->num_weights, sizeof(float));
252
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3461 times.
 6922 if (!weights)
253 ✗ return AVERROR(ENOMEM);
254
255
2/2
✓ Branch 0 taken 727872 times.
✓ Branch 1 taken 3461 times.
 1462666 for (int i = 0; i < filter->num_weights; i++)
256 1455744 weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE;
257
258 6922 out->priv.ptr = weights;
259 6922 out->priv.i32[2] = filter->filter_size;
260 6922 out->free = ff_op_priv_free;
261 6922 return 0;
262 }
263
264 /* Fully general vertical planar filter case */
265 1042688 DECL_READ(read_planar_fv, const SwsCompMask mask, const SwsPixelType type)
266 {
267 av_assert2(type == SWS_PIXEL_F32);
268 1042688 const SwsOpExec *exec = iter->exec;
269 1042688 const float *restrict weights = impl->priv.ptr;
270 1042688 const int filter_size = impl->priv.i32[2];
271 1042688 weights += filter_size * iter->y;
272
273 block_t xs, ys, zs, ws;
274
1/2
✓ Branch 0 taken 521344 times.
✗ Branch 1 not taken.
 1042688 if (X) memset(&xs.f32, 0, sizeof(xs.f32));
275
2/2
✓ Branch 0 taken 421824 times.
✓ Branch 1 taken 99520 times.
 1042688 if (Y) memset(&ys.f32, 0, sizeof(ys.f32));
276
2/2
✓ Branch 0 taken 406336 times.
✓ Branch 1 taken 115008 times.
 1042688 if (Z) memset(&zs.f32, 0, sizeof(zs.f32));
277
2/2
✓ Branch 0 taken 97152 times.
✓ Branch 1 taken 424192 times.
 1042688 if (W) memset(&ws.f32, 0, sizeof(ws.f32));
278
279
2/2
✓ Branch 0 taken 1947264 times.
✓ Branch 1 taken 521344 times.
 4937216 for (int j = 0; j < filter_size; j++) {
280 3894528 const float weight = weights[j];
281
282 SWS_LOOP
283
2/2
✓ Branch 0 taken 62312448 times.
✓ Branch 1 taken 1947264 times.
 128519424 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
284
1/2
✓ Branch 0 taken 62312448 times.
✗ Branch 1 not taken.
 124624896 if (X) xs.f32[i] += weight * in0[i];
285
2/2
✓ Branch 0 taken 51179520 times.
✓ Branch 1 taken 11132928 times.
 124624896 if (Y) ys.f32[i] += weight * in1[i];
286
2/2
✓ Branch 0 taken 49504256 times.
✓ Branch 1 taken 12808192 times.
 124624896 if (Z) zs.f32[i] += weight * in2[i];
287
2/2
✓ Branch 0 taken 11862016 times.
✓ Branch 1 taken 50450432 times.
 124624896 if (W) ws.f32[i] += weight * in3[i];
288 }
289
290
1/2
✓ Branch 0 taken 1947264 times.
✗ Branch 1 not taken.
 3894528 if (X) in0 = bump_ptr(in0, exec->in_stride[0]);
291
2/2
✓ Branch 0 taken 1599360 times.
✓ Branch 1 taken 347904 times.
 3894528 if (Y) in1 = bump_ptr(in1, exec->in_stride[1]);
292
2/2
✓ Branch 0 taken 1547008 times.
✓ Branch 1 taken 400256 times.
 3894528 if (Z) in2 = bump_ptr(in2, exec->in_stride[2]);
293
2/2
✓ Branch 0 taken 370688 times.
✓ Branch 1 taken 1576576 times.
 3894528 if (W) in3 = bump_ptr(in3, exec->in_stride[3]);
294 }
295
296
1/2
✓ Branch 0 taken 521344 times.
✗ Branch 1 not taken.
 1042688 if (X) iter->in[0] += SIZEOF_BLOCK;
297
2/2
✓ Branch 0 taken 421824 times.
✓ Branch 1 taken 99520 times.
 1042688 if (Y) iter->in[1] += SIZEOF_BLOCK;
298
2/2
✓ Branch 0 taken 406336 times.
✓ Branch 1 taken 115008 times.
 1042688 if (Z) iter->in[2] += SIZEOF_BLOCK;
299
2/2
✓ Branch 0 taken 97152 times.
✓ Branch 1 taken 424192 times.
 1042688 if (W) iter->in[3] += SIZEOF_BLOCK;
300
301 1042688 CONTINUE(&xs, &ys, &zs, &ws);
302 1042688 }
303
304 5852 DECL_SETUP(setup_filter_h, params, out)
305 {
306
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2926 times.
 5852 if (params->uop->par.filter.type != SWS_PIXEL_F32)
307 ✗ return AVERROR(ENOTSUP);
308
309 5852 SwsFilterWeights *filter = params->uop->data.kernel;
310 5852 out->priv.i32[2] = filter->filter_size;
311
312 /* The horizontal filter uop reads weights for a full SWS_BLOCK_SIZE
313 * outputs per block (see read_planar_fh), so the weights array must be
314 * padded up to the block-aligned output count. Pad the tail with zero
315 * weights, which contribute nothing to the accumulated sums. */
316 5852 const size_t padded_w = FFALIGN(filter->dst_size, SWS_BLOCK_SIZE);
317
1/2
✓ Branch 0 taken 2926 times.
✗ Branch 1 not taken.
 5852 if (padded_w == filter->dst_size) {
318 5852 out->priv.ptr = av_refstruct_ref(filter->weights);
319 5852 out->free = ff_op_priv_unref;
320 5852 return 0;
321 }
322
323 ✗ int *weights = av_calloc(padded_w * filter->filter_size, sizeof(*weights));
324 ✗ if (!weights)
325 ✗ return AVERROR(ENOMEM);
326 ✗ memcpy(weights, filter->weights, filter->num_weights * sizeof(*weights));
327 ✗ out->priv.ptr = weights;
328 ✗ out->free = ff_op_priv_free;
329 ✗ return 0;
330 }
331
332 /* Fully general horizontal planar filter case */
333 368256 DECL_READ(read_planar_fh, const SwsCompMask mask, const SwsPixelType type)
334 {
335 av_assert2(type == SWS_PIXEL_F32);
336 368256 const SwsOpExec *exec = iter->exec;
337 368256 const int *restrict weights = impl->priv.ptr;
338 368256 const int filter_size = impl->priv.i32[2];
339 368256 const float scale = 1.0f / SWS_FILTER_SCALE;
340 368256 const int xpos = iter->x;
341 368256 weights += filter_size * iter->x;
342
343 block_t xs, ys, zs, ws;
344
2/2
✓ Branch 0 taken 5892096 times.
✓ Branch 1 taken 184128 times.
 12152448 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
345 11784192 const int offset = exec->in_offset_x[xpos + i];
346 11784192 pixel_t *start0 = bump_ptr(in0, offset);
347 11784192 pixel_t *start1 = bump_ptr(in1, offset);
348 11784192 pixel_t *start2 = bump_ptr(in2, offset);
349 11784192 pixel_t *start3 = bump_ptr(in3, offset);
350
351 11784192 inter_t sx = 0, sy = 0, sz = 0, sw = 0;
352
2/2
✓ Branch 0 taken 22867968 times.
✓ Branch 1 taken 5892096 times.
 57520128 for (int j = 0; j < filter_size; j++) {
353 45735936 const int weight = weights[j];
354
1/2
✓ Branch 0 taken 22867968 times.
✗ Branch 1 not taken.
 45735936 if (X) sx += weight * start0[j];
355
2/2
✓ Branch 0 taken 17952768 times.
✓ Branch 1 taken 4915200 times.
 45735936 if (Y) sy += weight * start1[j];
356
2/2
✓ Branch 0 taken 16601088 times.
✓ Branch 1 taken 6266880 times.
 45735936 if (Z) sz += weight * start2[j];
357
2/2
✓ Branch 0 taken 3907584 times.
✓ Branch 1 taken 18960384 times.
 45735936 if (W) sw += weight * start3[j];
358 }
359
360
1/2
✓ Branch 0 taken 5892096 times.
✗ Branch 1 not taken.
 11784192 if (X) xs.f32[i] = (float) sx * scale;
361
2/2
✓ Branch 0 taken 4638720 times.
✓ Branch 1 taken 1253376 times.
 11784192 if (Y) ys.f32[i] = (float) sy * scale;
362
2/2
✓ Branch 0 taken 4337664 times.
✓ Branch 1 taken 1554432 times.
 11784192 if (Z) zs.f32[i] = (float) sz * scale;
363
2/2
✓ Branch 0 taken 1001472 times.
✓ Branch 1 taken 4890624 times.
 11784192 if (W) ws.f32[i] = (float) sw * scale;
364
365 11784192 weights += filter_size;
366 }
367
368 368256 CONTINUE(&xs, &ys, &zs, &ws);
369 368256 }
370
371 1042688 SWS_FOR(PX, READ_PLANAR_FV, DECL_IMPL_READ, read_planar_fv)
372 368256 SWS_FOR(PX, READ_PLANAR_FH, DECL_IMPL_READ, read_planar_fh)
373 SWS_FOR_STRUCT(PX, READ_PLANAR_FV, DECL_ENTRY, .setup = fn(setup_filter_v) )
374 SWS_FOR_STRUCT(PX, READ_PLANAR_FH, DECL_ENTRY, .setup = fn(setup_filter_h) )
375
376 /***************************
377 * Permutation and copying *
378 ***************************/
379
380 2657984 DECL_FUNC(permute, const SwsCompMask mask, int num_moves,
381 int8_t d0, int8_t d1, int8_t d2, int8_t d3, int8_t d4, int8_t d5,
382 int8_t s0, int8_t s1, int8_t s2, int8_t s3, int8_t s4, int8_t s5)
383 {
384 2657984 const int8_t dst[SWS_UOP_MOVE_MAX] = { d0, d1, d2, d3, d4, d5 };
385 2657984 const int8_t src[SWS_UOP_MOVE_MAX] = { s0, s1, s2, s3, s4, s5 };
386
387 2657984 pixel_t *ptr[5] = { NULL, x, y, z, w };
388
2/2
✓ Branch 0 taken 4377984 times.
✓ Branch 1 taken 1328992 times.
 11413952 for (int n = 0; n < num_moves; n++)
389 8755968 ptr[dst[n] + 1] = ptr[src[n] + 1];
390
391 /* The unneeded registers may still alias the used ones, so point them
392 * back at the stack to avoid collisions */
393 block_t xx, yy, zz, ww;
394
8/8
✓ Branch 0 taken 1013120 times.
✓ Branch 1 taken 315872 times.
✓ Branch 2 taken 1316320 times.
✓ Branch 3 taken 12672 times.
✓ Branch 4 taken 1299584 times.
✓ Branch 5 taken 29408 times.
✓ Branch 6 taken 929792 times.
✓ Branch 7 taken 399200 times.
 2657984 CONTINUE(X ? ptr[1] : xx.px,
395 Y ? ptr[2] : yy.px,
396 Z ? ptr[3] : zz.px,
397 W ? ptr[4] : ww.px);
398 2657984 }
399
400 175104 DECL_FUNC(copy, const SwsCompMask mask, int num_moves,
401 int8_t d0, int8_t d1, int8_t d2, int8_t d3, int8_t d4, int8_t d5,
402 int8_t s0, int8_t s1, int8_t s2, int8_t s3, int8_t s4, int8_t s5)
403 {
404 175104 const size_t block_size = SWS_BLOCK_SIZE * sizeof(pixel_t);
405 175104 const int8_t dst[SWS_UOP_MOVE_MAX] = { d0, d1, d2, d3, d4, d5 };
406 175104 const int8_t src[SWS_UOP_MOVE_MAX] = { s0, s1, s2, s3, s4, s5 };
407
408 block_t data[5];
409 175104 memcpy(&data[1].px, x, block_size);
410 175104 memcpy(&data[2].px, y, block_size);
411 175104 memcpy(&data[3].px, z, block_size);
412 175104 memcpy(&data[4].px, w, block_size);
413
414
2/2
✓ Branch 0 taken 223360 times.
✓ Branch 1 taken 87552 times.
 621824 for (int n = 0; n < num_moves; n++)
415 446720 data[dst[n] + 1] = data[src[n] + 1];
416
417 175104 memcpy(x, &data[1].px, block_size);
418 175104 memcpy(y, &data[2].px, block_size);
419 175104 memcpy(z, &data[3].px, block_size);
420 175104 memcpy(w, &data[4].px, block_size);
421
422 175104 CONTINUE(x, y, z, w);
423 175104 }
424
425 2657984 SWS_FOR(PX, PERMUTE, DECL_IMPL, permute)
426 175104 SWS_FOR(PX, COPY, DECL_IMPL, copy)
427 SWS_FOR_STRUCT(PX, PERMUTE, DECL_ENTRY)
428 SWS_FOR_STRUCT(PX, COPY, DECL_ENTRY)
429
430 /*********************
431 * Format conversion *
432 *********************/
433
434 #define DECL_CAST(DST, dst) \
435 DECL_FUNC(to_##dst, const SwsCompMask mask) \
436 { \
437 block_t xx, yy, zz, ww; \
438 \
439 SWS_LOOP \
440 for (int i = 0; i < SWS_BLOCK_SIZE; i++) { \
441 if (X) xx.dst[i] = x[i]; \
442 if (Y) yy.dst[i] = y[i]; \
443 if (Z) zz.dst[i] = z[i]; \
444 if (W) ww.dst[i] = w[i]; \
445 } \
446 \
447 CONTINUE(&xx, &yy, &zz, &ww); \
448 } \
449 \
450 SWS_FOR(PX, TO_##DST, DECL_IMPL, to_##dst) \
451 SWS_FOR_STRUCT(PX, TO_##DST, DECL_ENTRY)
452
453
10/10
✓ Branch 0 taken 6066176 times.
✓ Branch 1 taken 1792000 times.
✓ Branch 2 taken 7436288 times.
✓ Branch 3 taken 421888 times.
✓ Branch 4 taken 7432192 times.
✓ Branch 5 taken 425984 times.
✓ Branch 6 taken 1937408 times.
✓ Branch 7 taken 5920768 times.
✓ Branch 8 taken 7858176 times.
✓ Branch 9 taken 245568 times.
 16698624 DECL_CAST(U8, u8)
454
10/10
✓ Branch 0 taken 7356416 times.
✓ Branch 1 taken 1273856 times.
✓ Branch 2 taken 5853184 times.
✓ Branch 3 taken 2777088 times.
✓ Branch 4 taken 5849088 times.
✓ Branch 5 taken 2781184 times.
✓ Branch 6 taken 1427456 times.
✓ Branch 7 taken 7202816 times.
✓ Branch 8 taken 8630272 times.
✓ Branch 9 taken 269696 times.
 18339328 DECL_CAST(U16, u16)
455
10/10
✓ Branch 0 taken 21358592 times.
✓ Branch 1 taken 76800 times.
✓ Branch 2 taken 18562048 times.
✓ Branch 3 taken 2873344 times.
✓ Branch 4 taken 18562048 times.
✓ Branch 5 taken 2873344 times.
✓ Branch 6 taken 7099392 times.
✓ Branch 7 taken 14336000 times.
✓ Branch 8 taken 21435392 times.
✓ Branch 9 taken 669856 times.
 45550208 DECL_CAST(U32, u32)
456
10/10
✓ Branch 0 taken 14228480 times.
✓ Branch 1 taken 10820608 times.
✓ Branch 2 taken 22310912 times.
✓ Branch 3 taken 2738176 times.
✓ Branch 4 taken 22003712 times.
✓ Branch 5 taken 3045376 times.
✓ Branch 6 taken 12321792 times.
✓ Branch 7 taken 12727296 times.
✓ Branch 8 taken 25049088 times.
✓ Branch 9 taken 782784 times.
 53229312 DECL_CAST(F32, f32)
457
458 /********************
459 * Bit manipulation *
460 ********************/
461
462 #if !IS_FLOAT
463 109696 DECL_FUNC(lshift, const SwsCompMask mask, const uint8_t amount)
464 {
465 SWS_LOOP
466
2/2
✓ Branch 0 taken 1755136 times.
✓ Branch 1 taken 54848 times.
 3619968 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
467
2/2
✓ Branch 0 taken 1660928 times.
✓ Branch 1 taken 94208 times.
 3510272 if (X) x[i] <<= amount;
468
2/2
✓ Branch 0 taken 884736 times.
✓ Branch 1 taken 870400 times.
 3510272 if (Y) y[i] <<= amount;
469
2/2
✓ Branch 0 taken 884736 times.
✓ Branch 1 taken 870400 times.
 3510272 if (Z) z[i] <<= amount;
470
2/2
✓ Branch 0 taken 94208 times.
✓ Branch 1 taken 1660928 times.
 3510272 if (W) w[i] <<= amount;
471 }
472
473 109696 CONTINUE(x, y, z, w);
474 109696 }
475
476 38080 DECL_FUNC(rshift, const SwsCompMask mask, const uint8_t amount)
477 {
478 SWS_LOOP
479
2/2
✓ Branch 0 taken 1218560 times.
✓ Branch 1 taken 38080 times.
 1256640 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
480
2/2
✓ Branch 0 taken 1177600 times.
✓ Branch 1 taken 40960 times.
 1218560 if (X) x[i] >>= amount;
481
2/2
✓ Branch 0 taken 1136640 times.
✓ Branch 1 taken 81920 times.
 1218560 if (Y) y[i] >>= amount;
482
2/2
✓ Branch 0 taken 1095680 times.
✓ Branch 1 taken 122880 times.
 1218560 if (Z) z[i] >>= amount;
483
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1218560 times.
 1218560 if (W) w[i] >>= amount;
484 }
485
486 38080 CONTINUE(x, y, z, w);
487 38080 }
488 #endif
489
490 109696 SWS_FOR(PX, LSHIFT, DECL_IMPL, lshift)
491 76160 SWS_FOR(PX, RSHIFT, DECL_IMPL, rshift)
492
493 SWS_FOR_STRUCT(PX, LSHIFT, DECL_ENTRY)
494 SWS_FOR_STRUCT(PX, RSHIFT, DECL_ENTRY)
495
496 #ifdef PIXEL_SWAP
497 2225024 DECL_FUNC(swap_bytes, const SwsCompMask mask)
498 {
499 SWS_LOOP
500
2/2
✓ Branch 0 taken 35600384 times.
✓ Branch 1 taken 1112512 times.
 73425792 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
501
2/2
✓ Branch 0 taken 34998272 times.
✓ Branch 1 taken 602112 times.
 71200768 if (X) x[i] = PIXEL_SWAP(x[i]);
502
2/2
✓ Branch 0 taken 11632640 times.
✓ Branch 1 taken 23967744 times.
 71200768 if (Y) y[i] = PIXEL_SWAP(y[i]);
503
2/2
✓ Branch 0 taken 9627648 times.
✓ Branch 1 taken 25972736 times.
 71200768 if (Z) z[i] = PIXEL_SWAP(z[i]);
504
2/2
✓ Branch 0 taken 5736448 times.
✓ Branch 1 taken 29863936 times.
 71200768 if (W) w[i] = PIXEL_SWAP(w[i]);
505 }
506
507 2225024 CONTINUE(x, y, z, w);
508 2225024 }
509 #endif /* PIXEL_SWAP */
510
511 2225024 SWS_FOR(PX, SWAP_BYTES, DECL_IMPL, swap_bytes)
512 SWS_FOR_STRUCT(PX, SWAP_BYTES, DECL_ENTRY)
513
514 #ifdef PIXEL_MAX
515 576 DECL_FUNC(expand_bit, const SwsCompMask mask)
516 {
517 SWS_LOOP
518
2/2
✓ Branch 0 taken 9216 times.
✓ Branch 1 taken 288 times.
 19008 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
519
3/4
✓ Branch 0 taken 9216 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4707 times.
✓ Branch 3 taken 4509 times.
 18432 if (X) x[i] = x[i] ? PIXEL_MAX : 0;
520
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 9216 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 18432 if (Y) y[i] = y[i] ? PIXEL_MAX : 0;
521
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 9216 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 18432 if (Z) z[i] = z[i] ? PIXEL_MAX : 0;
522
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 9216 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 18432 if (W) w[i] = w[i] ? PIXEL_MAX : 0;
523 }
524
525 576 CONTINUE(x, y, z, w);
526 576 }
527 #endif
528
529 #if BIT_DEPTH == 8
530 640 DECL_FUNC(expand_pair, const SwsCompMask mask)
531 {
532 block_t x16, y16, z16, w16;
533
534 SWS_LOOP
535
2/2
✓ Branch 0 taken 20480 times.
✓ Branch 1 taken 640 times.
 21120 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
536
2/2
✓ Branch 0 taken 16384 times.
✓ Branch 1 taken 4096 times.
 20480 if (X) x16.u16[i] = x[i] << 8 | x[i];
537
2/2
✓ Branch 0 taken 16384 times.
✓ Branch 1 taken 4096 times.
 20480 if (Y) y16.u16[i] = y[i] << 8 | y[i];
538
2/2
✓ Branch 0 taken 12288 times.
✓ Branch 1 taken 8192 times.
 20480 if (Z) z16.u16[i] = z[i] << 8 | z[i];
539
2/2
✓ Branch 0 taken 8192 times.
✓ Branch 1 taken 12288 times.
 20480 if (W) w16.u16[i] = w[i] << 8 | w[i];
540 }
541
542 640 CONTINUE(&x16, &y16, &z16, &w16);
543 640 }
544
545 DECL_FUNC(expand_quad, const SwsCompMask mask)
546 {
547 block_t x32, y32, z32, w32;
548
549 SWS_LOOP
550 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
551 if (X) x32.u32[i] = (uint32_t) x[i] << 24 | x[i] << 16 | x[i] << 8 | x[i];
552 if (Y) y32.u32[i] = (uint32_t) y[i] << 24 | y[i] << 16 | y[i] << 8 | y[i];
553 if (Z) z32.u32[i] = (uint32_t) z[i] << 24 | z[i] << 16 | z[i] << 8 | z[i];
554 if (W) w32.u32[i] = (uint32_t) w[i] << 24 | w[i] << 16 | w[i] << 8 | w[i];
555 }
556
557 CONTINUE(&x32, &y32, &z32, &w32);
558 }
559 #endif /* BIT_DEPTH == 8 */
560
561 576 SWS_FOR(PX, EXPAND_BIT, DECL_IMPL, expand_bit)
562 1280 SWS_FOR(PX, EXPAND_PAIR, DECL_IMPL, expand_pair)
563 SWS_FOR(PX, EXPAND_QUAD, DECL_IMPL, expand_quad)
564 SWS_FOR_STRUCT(PX, EXPAND_BIT, DECL_ENTRY)
565 SWS_FOR_STRUCT(PX, EXPAND_PAIR, DECL_ENTRY)
566 SWS_FOR_STRUCT(PX, EXPAND_QUAD, DECL_ENTRY)
567
568 /*************************
569 * Packing and unpacking *
570 ************************/
571
572 #if !IS_FLOAT
573 976192 DECL_FUNC(unpack, const SwsCompMask mask,
574 const uint8_t bx, const uint8_t by,
575 const uint8_t bz, const uint8_t bw)
576 {
577 976192 const uint8_t sx = bw + bz + by;
578 976192 const uint8_t sy = bw + bz;
579 976192 const uint8_t sz = bw;
580 976192 const uint8_t sw = 0;
581
582 976192 const pixel_t mx = (1 << bx) - 1;
583 976192 const pixel_t my = (1 << by) - 1;
584 976192 const pixel_t mz = (1 << bz) - 1;
585 976192 const pixel_t mw = (1 << bw) - 1;
586
587 SWS_LOOP
588
2/2
✓ Branch 0 taken 15619072 times.
✓ Branch 1 taken 488096 times.
 32214336 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
589 31238144 const pixel_t val = x[i];
590
2/2
✓ Branch 0 taken 5008384 times.
✓ Branch 1 taken 10610688 times.
 31238144 if (X) x[i] = (val >> sx) & mx;
591
2/2
✓ Branch 0 taken 15246336 times.
✓ Branch 1 taken 372736 times.
 31238144 if (Y) y[i] = (val >> sy) & my;
592
2/2
✓ Branch 0 taken 15246336 times.
✓ Branch 1 taken 372736 times.
 31238144 if (Z) z[i] = (val >> sz) & mz;
593
2/2
✓ Branch 0 taken 9865216 times.
✓ Branch 1 taken 5753856 times.
 31238144 if (W) w[i] = (val >> sw) & mw;
594 }
595
596 976192 CONTINUE(x, y, z, w);
597 976192 }
598
599 1466048 DECL_FUNC(pack, const SwsCompMask mask,
600 const uint8_t bx, const uint8_t by,
601 const uint8_t bz, const uint8_t bw)
602 {
603 1466048 const uint8_t sx = bw + bz + by;
604 1466048 const uint8_t sy = bw + bz;
605 1466048 const uint8_t sz = bw;
606 1466048 const uint8_t sw = 0;
607
608 SWS_LOOP
609
2/2
✓ Branch 0 taken 23456768 times.
✓ Branch 1 taken 733024 times.
 48379584 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
610 46913536 pixel_t val = 0;
611
1/2
✓ Branch 0 taken 23456768 times.
✗ Branch 1 not taken.
 46913536 if (X) val |= x[i] << sx;
612
1/2
✓ Branch 0 taken 23456768 times.
✗ Branch 1 not taken.
 46913536 if (Y) val |= y[i] << sy;
613
1/2
✓ Branch 0 taken 23456768 times.
✗ Branch 1 not taken.
 46913536 if (Z) val |= z[i] << sz;
614
2/2
✓ Branch 0 taken 21633024 times.
✓ Branch 1 taken 1823744 times.
 46913536 if (W) val |= w[i] << sw;
615 46913536 x[i] = val;
616 }
617
618 1466048 CONTINUE(x, y, z, w);
619 1466048 }
620 #endif /* !IS_FLOAT */
621
622 976192 SWS_FOR(PX, UNPACK, DECL_IMPL, unpack)
623 1466048 SWS_FOR(PX, PACK, DECL_IMPL, pack)
624 SWS_FOR_STRUCT(PX, UNPACK, DECL_ENTRY)
625 SWS_FOR_STRUCT(PX, PACK, DECL_ENTRY)
626
627 /***********************
628 * Pixel data clearing *
629 ***********************/
630
631 #ifdef PIXEL_MAX
632 3366464 DECL_FUNC(clear, const SwsCompMask mask, const SwsCompMask one,
633 const SwsCompMask zero)
634 {
635 #define ONE(N) SWS_COMP_TEST(one, N)
636 #define ZERO(N) SWS_COMP_TEST(zero, N)
637
4/4
✓ Branch 0 taken 1669088 times.
✓ Branch 1 taken 14144 times.
✓ Branch 2 taken 1662080 times.
✓ Branch 3 taken 7008 times.
 3366464 const pixel_t cx = ONE(0) ? PIXEL_MAX : ZERO(0) ? 0 : impl->priv.px[0];
638
3/4
✓ Branch 0 taken 1676000 times.
✓ Branch 1 taken 7232 times.
✓ Branch 2 taken 1676000 times.
✗ Branch 3 not taken.
 3366464 const pixel_t cy = ONE(1) ? PIXEL_MAX : ZERO(1) ? 0 : impl->priv.px[1];
639
2/4
✓ Branch 0 taken 1683232 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1683232 times.
✗ Branch 3 not taken.
 3366464 const pixel_t cz = ONE(2) ? PIXEL_MAX : ZERO(2) ? 0 : impl->priv.px[2];
640
4/4
✓ Branch 0 taken 1660576 times.
✓ Branch 1 taken 22656 times.
✓ Branch 2 taken 1586016 times.
✓ Branch 3 taken 74560 times.
 3366464 const pixel_t cw = ONE(3) ? PIXEL_MAX : ZERO(3) ? 0 : impl->priv.px[3];
641
642 SWS_LOOP
643
2/2
✓ Branch 0 taken 53863424 times.
✓ Branch 1 taken 1683232 times.
 111093312 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
644
2/2
✓ Branch 0 taken 38715392 times.
✓ Branch 1 taken 15148032 times.
 107726848 if (X) x[i] = cx;
645
2/2
✓ Branch 0 taken 6843392 times.
✓ Branch 1 taken 47020032 times.
 107726848 if (Y) y[i] = cy;
646
2/2
✓ Branch 0 taken 4703232 times.
✓ Branch 1 taken 49160192 times.
 107726848 if (Z) z[i] = cz;
647
2/2
✓ Branch 0 taken 14279680 times.
✓ Branch 1 taken 39583744 times.
 107726848 if (W) w[i] = cw;
648 }
649
650 3366464 CONTINUE(x, y, z, w);
651 3366464 }
652 #endif
653
654 3366464 SWS_FOR(PX, CLEAR, DECL_IMPL, clear)
655 SWS_FOR_STRUCT(PX, CLEAR, DECL_ENTRY, .setup = ff_sws_setup_vec4)
656
657 /*************************
658 * Arithmetic operations *
659 *************************/
660
661 831936 DECL_FUNC(scale, const SwsCompMask mask)
662 {
663 831936 const pixel_t scale = impl->priv.px[0];
664
665 SWS_LOOP
666
2/2
✓ Branch 0 taken 13310976 times.
✓ Branch 1 taken 415968 times.
 27453888 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
667
2/2
✓ Branch 0 taken 9841664 times.
✓ Branch 1 taken 3469312 times.
 26621952 if (X) x[i] *= scale;
668
2/2
✓ Branch 0 taken 11132928 times.
✓ Branch 1 taken 2178048 times.
 26621952 if (Y) y[i] *= scale;
669
2/2
✓ Branch 0 taken 11128832 times.
✓ Branch 1 taken 2182144 times.
 26621952 if (Z) z[i] *= scale;
670
2/2
✓ Branch 0 taken 3473408 times.
✓ Branch 1 taken 9837568 times.
 26621952 if (W) w[i] *= scale;
671 }
672
673 831936 CONTINUE(x, y, z, w);
674 831936 }
675
676 5504 DECL_FUNC(add, const SwsCompMask mask)
677 {
678 SWS_LOOP
679
2/2
✓ Branch 0 taken 176128 times.
✓ Branch 1 taken 5504 times.
 181632 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
680
2/2
✓ Branch 0 taken 167936 times.
✓ Branch 1 taken 8192 times.
 176128 if (X) x[i] += impl->priv.px[0];
681
2/2
✓ Branch 0 taken 16384 times.
✓ Branch 1 taken 159744 times.
 176128 if (Y) y[i] += impl->priv.px[1];
682
2/2
✓ Branch 0 taken 16384 times.
✓ Branch 1 taken 159744 times.
 176128 if (Z) z[i] += impl->priv.px[2];
683
2/2
✓ Branch 0 taken 16384 times.
✓ Branch 1 taken 159744 times.
 176128 if (W) w[i] += impl->priv.px[3];
684 }
685
686 5504 CONTINUE(x, y, z, w);
687 5504 }
688
689 804160 DECL_FUNC(min, const SwsCompMask mask)
690 {
691 SWS_LOOP
692
2/2
✓ Branch 0 taken 25733120 times.
✓ Branch 1 taken 804160 times.
 26537280 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
693
2/2
✓ Branch 0 taken 23230464 times.
✓ Branch 1 taken 2502656 times.
 25733120 if (X) x[i] = FFMIN(x[i], impl->priv.px[0]);
694
2/2
✓ Branch 0 taken 21940224 times.
✓ Branch 1 taken 3792896 times.
 25733120 if (Y) y[i] = FFMIN(y[i], impl->priv.px[1]);
695
2/2
✓ Branch 0 taken 21673984 times.
✓ Branch 1 taken 4059136 times.
 25733120 if (Z) z[i] = FFMIN(z[i], impl->priv.px[2]);
696
2/2
✓ Branch 0 taken 2611200 times.
✓ Branch 1 taken 23121920 times.
 25733120 if (W) w[i] = FFMIN(w[i], impl->priv.px[3]);
697 }
698
699 804160 CONTINUE(x, y, z, w);
700 804160 }
701
702 530688 DECL_FUNC(max, const SwsCompMask mask)
703 {
704 SWS_LOOP
705
2/2
✓ Branch 0 taken 16982016 times.
✓ Branch 1 taken 530688 times.
 17512704 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
706
2/2
✓ Branch 0 taken 16977920 times.
✓ Branch 1 taken 4096 times.
 16982016 if (X) x[i] = FFMAX(x[i], impl->priv.px[0]);
707
2/2
✓ Branch 0 taken 13991936 times.
✓ Branch 1 taken 2990080 times.
 16982016 if (Y) y[i] = FFMAX(y[i], impl->priv.px[1]);
708
2/2
✓ Branch 0 taken 13987840 times.
✓ Branch 1 taken 2994176 times.
 16982016 if (Z) z[i] = FFMAX(z[i], impl->priv.px[2]);
709
2/2
✓ Branch 0 taken 3508224 times.
✓ Branch 1 taken 13473792 times.
 16982016 if (W) w[i] = FFMAX(w[i], impl->priv.px[3]);
710 }
711
712 530688 CONTINUE(x, y, z, w);
713 530688 }
714
715 831936 SWS_FOR(PX, SCALE, DECL_IMPL, scale)
716 11008 SWS_FOR(PX, ADD, DECL_IMPL, add)
717 1608320 SWS_FOR(PX, MIN, DECL_IMPL, min)
718 1061376 SWS_FOR(PX, MAX, DECL_IMPL, max)
719 SWS_FOR_STRUCT(PX, SCALE, DECL_ENTRY, .setup = ff_sws_setup_scalar )
720 SWS_FOR_STRUCT(PX, ADD, DECL_ENTRY, .setup = ff_sws_setup_vec4 )
721 SWS_FOR_STRUCT(PX, MIN, DECL_ENTRY, .setup = ff_sws_setup_vec4 )
722 SWS_FOR_STRUCT(PX, MAX, DECL_ENTRY, .setup = ff_sws_setup_vec4 )
723
724 /*************
725 * Dithering *
726 *************/
727
728 7460 DECL_SETUP(setup_dither, params, out)
729 {
730 7460 const SwsUOp *uop = params->uop;
731 7460 const SwsDitherUOp *dither = &uop->par.dither;
732 7460 const int size = 1 << dither->size_log2;
733
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3730 times.
 7460 if (size >= SWS_BLOCK_SIZE) {
734 /* No extra padding needed */
735 ✗ out->priv.ptr = av_refstruct_ref(uop->data.ptr);
736 ✗ out->free = ff_op_priv_unref;
737 ✗ return 0;
738 }
739
740 7460 const int stride = FFMAX(size, SWS_BLOCK_SIZE);
741 7460 const int height = ff_sws_dither_height(dither);
742 7460 pixel_t *matrix = av_malloc(sizeof(pixel_t) * height * stride);
743
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3730 times.
 7460 if (!matrix)
744 ✗ return AVERROR(ENOMEM);
745 7460 out->priv.ptr = matrix;
746 7460 out->free = ff_op_priv_free;
747
748 /* Pad to multiple of block size. We don't need extra padding for the
749 * height because ff_sws_dither_height() already includes any padding
750 * necessary for the y_offset */
751
2/2
✓ Branch 0 taken 71437 times.
✓ Branch 1 taken 3730 times.
 150334 for (int y = 0; y < height; y++) {
752 142874 pixel_t *row = &matrix[y * stride];
753
2/2
✓ Branch 0 taken 1142992 times.
✓ Branch 1 taken 71437 times.
 2428858 for (int x = 0; x < size; x++)
754 2285984 row[x] = uop->data.ptr[y * size + x].px;
755
2/2
✓ Branch 0 taken 1142992 times.
✓ Branch 1 taken 71437 times.
 2428858 for (int x = size; x < stride; x++)
756 2285984 row[x] = row[x % size];
757 }
758
759 7460 return 0;
760 }
761
762 970688 DECL_FUNC(dither, const SwsCompMask mask,
763 const uint8_t off0, const uint8_t off1,
764 const uint8_t off2, const uint8_t off3,
765 const uint8_t size_log2)
766 {
767 970688 const int size = 1 << size_log2;
768 970688 const int stride = FFMAX(size, SWS_BLOCK_SIZE);
769
770 970688 const pixel_t *matrix = impl->priv.ptr;
771 970688 matrix += (iter->y & (size - 1)) * stride;
772 970688 matrix += (iter->x & (size - 1)) & ~(SWS_BLOCK_SIZE - 1);
773
774 970688 const pixel_t *const row0 = &matrix[off0 * stride];
775 970688 const pixel_t *const row1 = &matrix[off1 * stride];
776 970688 const pixel_t *const row2 = &matrix[off2 * stride];
777 970688 const pixel_t *const row3 = &matrix[off3 * stride];
778
779 SWS_LOOP
780
2/2
✓ Branch 0 taken 31062016 times.
✓ Branch 1 taken 970688 times.
 32032704 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
781
2/2
✓ Branch 0 taken 28057600 times.
✓ Branch 1 taken 3004416 times.
 31062016 if (X) x[i] += row0[i];
782
2/2
✓ Branch 0 taken 27187200 times.
✓ Branch 1 taken 3874816 times.
 31062016 if (Y) y[i] += row1[i];
783
2/2
✓ Branch 0 taken 26920960 times.
✓ Branch 1 taken 4141056 times.
 31062016 if (Z) z[i] += row2[i];
784
2/2
✓ Branch 0 taken 9529344 times.
✓ Branch 1 taken 21532672 times.
 31062016 if (W) w[i] += row3[i];
785 }
786
787 970688 CONTINUE(x, y, z, w);
788 970688 }
789
790 1941376 SWS_FOR(PX, DITHER, DECL_IMPL, dither)
791 SWS_FOR_STRUCT(PX, DITHER, DECL_ENTRY, .setup = fn(setup_dither) )
792
793 /*********************
794 * Linear operations *
795 *********************/
796
797 typedef struct {
798 /* Stored in split form for convenience */
799 pixel_t m[4][4];
800 pixel_t k[4];
801 } fn(LinCoeffs);
802
803 7734 DECL_SETUP(setup_linear, params, out)
804 {
805 7734 const SwsUOp *uop = params->uop;
806 fn(LinCoeffs) c;
807
808
2/2
✓ Branch 0 taken 15468 times.
✓ Branch 1 taken 3867 times.
 38670 for (int i = 0; i < 4; i++) {
809
2/2
✓ Branch 0 taken 61872 times.
✓ Branch 1 taken 15468 times.
 154680 for (int j = 0; j < 4; j++)
810 123744 c.m[i][j] = uop->data.mat4[i][j].px;
811 30936 c.k[i] = uop->data.mat4[i][4].px;
812 }
813
814 7734 out->priv.ptr = av_memdup(&c, sizeof(c));
815 7734 out->free = ff_op_priv_free;
816
1/2
✓ Branch 0 taken 3867 times.
✗ Branch 1 not taken.
 7734 return out->priv.ptr ? 0 : AVERROR(ENOMEM);
817 }
818
819 /**
820 * Fully general case for a 5x5 linear affine transformation. Should never be
821 * called without constant `mask`. This function will compile down to the
822 * appropriately optimized version for the required subset of operations when
823 * called with a constant mask.
824 */
825 1044192 DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zero)
826 {
827 1044192 const fn(LinCoeffs) c = *(const fn(LinCoeffs) *) impl->priv.ptr;
828
829 SWS_LOOP
830
2/2
✓ Branch 0 taken 33414144 times.
✓ Branch 1 taken 1044192 times.
 34458336 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
831 33414144 const pixel_t xx = x[i];
832 33414144 const pixel_t yy = y[i];
833 33414144 const pixel_t zz = z[i];
834 33414144 const pixel_t ww = w[i];
835
836 #define LIN_VAL(I, J, val) \
837 ((one & SWS_MASK(I, J)) ? (val) : c.m[I][J] * (val))
838
839 #define LIN_ROW(I, var) do { \
840 var[i] = (zero & SWS_MASK(I, 4)) ? 0 : c.k[I]; \
841 if (!(zero & SWS_MASK(I, 0))) var[i] += LIN_VAL(I, 0, xx); \
842 if (!(zero & SWS_MASK(I, 1))) var[i] += LIN_VAL(I, 1, yy); \
843 if (!(zero & SWS_MASK(I, 2))) var[i] += LIN_VAL(I, 2, zz); \
844 if (!(zero & SWS_MASK(I, 3))) var[i] += LIN_VAL(I, 3, ww); \
845 } while (0)
846
847
14/20
✓ Branch 0 taken 32846848 times.
✓ Branch 1 taken 567296 times.
✓ Branch 2 taken 25508864 times.
✓ Branch 3 taken 7337984 times.
✓ Branch 4 taken 32846848 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 32842752 times.
✓ Branch 7 taken 4096 times.
✓ Branch 8 taken 13572096 times.
✓ Branch 9 taken 19274752 times.
✓ Branch 10 taken 13572096 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 22710272 times.
✓ Branch 13 taken 10136576 times.
✓ Branch 14 taken 22710272 times.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 32846848 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 33414144 if (X) LIN_ROW(0, x);
848
14/20
✓ Branch 0 taken 24757248 times.
✓ Branch 1 taken 8656896 times.
✓ Branch 2 taken 20042752 times.
✓ Branch 3 taken 4714496 times.
✓ Branch 4 taken 19779584 times.
✓ Branch 5 taken 4977664 times.
✓ Branch 6 taken 19775488 times.
✓ Branch 7 taken 4096 times.
✓ Branch 8 taken 24757248 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 24757248 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 19779584 times.
✓ Branch 13 taken 4977664 times.
✓ Branch 14 taken 19779584 times.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 24757248 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 33414144 if (Y) LIN_ROW(1, y);
849
15/20
✓ Branch 0 taken 24474624 times.
✓ Branch 1 taken 8939520 times.
✓ Branch 2 taken 20042752 times.
✓ Branch 3 taken 4431872 times.
✓ Branch 4 taken 19779584 times.
✓ Branch 5 taken 4695040 times.
✓ Branch 6 taken 19775488 times.
✓ Branch 7 taken 4096 times.
✓ Branch 8 taken 19779584 times.
✓ Branch 9 taken 4695040 times.
✓ Branch 10 taken 19779584 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 15336448 times.
✓ Branch 13 taken 9138176 times.
✓ Branch 14 taken 15336448 times.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 24474624 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 33414144 if (Z) LIN_ROW(2, z);
850
8/20
✓ Branch 0 taken 6297600 times.
✓ Branch 1 taken 27116544 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6297600 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 6297600 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 6297600 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 6297600 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 6297600 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 6297600 times.
✗ Branch 19 not taken.
 33414144 if (W) LIN_ROW(3, w);
851 }
852
853 1044192 CONTINUE(x, y, z, w);
854 1044192 }
855
856 2088384 SWS_FOR(PX, LINEAR, DECL_IMPL, linear)
857 SWS_FOR_STRUCT(PX, LINEAR, DECL_ENTRY, .setup = fn(setup_linear) )
858
859 /******************
860 * Look-up tables *
861 ******************/
862
863 112 DECL_SETUP(setup_lut3d, params, out)
864 {
865 112 const SwsLut3D *lut = params->uop->data.lut3d;
866 112 out->priv.ptr = (void *) av_refstruct_ref_c(lut);
867 112 out->free = ff_op_priv_unref;
868 112 return 0;
869 }
870
871 #if IS_FLOAT
872 98304 av_always_inline static vec3_t fn(vec3)(v3u16_t v)
873 {
874 98304 return (vec3_t) { v.x, v.y, v.z };
875 }
876
877 #define lerp(a, b, w) ((a) + (w) * ((pixel_t) (b) - (a)))
878
879 av_always_inline static
880 43008 vec3_t fn(lerp3)(vec3_t a, vec3_t b, pixel_t w)
881 {
882 86016 return (vec3_t) {
883 43008 lerp(a.x, b.x, w),
884 43008 lerp(a.y, b.y, w),
885 43008 lerp(a.z, b.z, w),
886 };
887 }
888
889 av_always_inline static
890 12288 vec3_t fn(lut3d_static)(const SwsLut3D *restrict lut3d, vec3_t rgb)
891 {
892 12288 const int r_base = (int) rgb.x;
893 12288 const int g_base = (int) rgb.y;
894 12288 const int b_base = (int) rgb.z;
895
896 12288 int off0 = (r_base < INPUT_LUT_SIZE - 1);
897
1/2
✓ Branch 0 taken 12288 times.
✗ Branch 1 not taken.
 12288 int off1 = (g_base < INPUT_LUT_SIZE - 1) * INPUT_LUT_SIZE;
898
1/2
✓ Branch 0 taken 12288 times.
✗ Branch 1 not taken.
 12288 int off2 = (b_base < INPUT_LUT_SIZE - 1) * INPUT_LUT_SIZE * INPUT_LUT_SIZE;
899 12288 pixel_t f0 = rgb.x - r_base;
900 12288 pixel_t f1 = rgb.y - g_base;
901 12288 pixel_t f2 = rgb.z - b_base;
902
903 /* Sort offsets descending by relative weight */
904
2/2
✓ Branch 0 taken 6216 times.
✓ Branch 1 taken 6072 times.
 12288 if (f0 < f1) {
905 6216 FFSWAP(pixel_t, f0, f1);
906 6216 FFSWAP(int, off0, off1);
907 }
908
2/2
✓ Branch 0 taken 3960 times.
✓ Branch 1 taken 8328 times.
 12288 if (f0 < f2) {
909 3960 FFSWAP(pixel_t, f0, f2);
910 3960 FFSWAP(int, off0, off2);
911 }
912
2/2
✓ Branch 0 taken 7908 times.
✓ Branch 1 taken 4380 times.
 12288 if (f1 < f2) {
913 7908 FFSWAP(pixel_t, f1, f2);
914 7908 FFSWAP(int, off1, off2);
915 }
916
917 /* Tetrahedral interpolation */
918 12288 const pixel_t w0 = 1 - f0;
919 12288 const pixel_t w1 = f0 - f1;
920 12288 const pixel_t w2 = f1 - f2;
921 12288 const pixel_t w3 = f2;
922
923 12288 const v3u16_t *restrict base = &lut3d->input[b_base][g_base][r_base];
924 12288 const vec3_t v0 = fn(vec3)(base[0]);
925 12288 const vec3_t v1 = fn(vec3)(base[off0]);
926 12288 const vec3_t v2 = fn(vec3)(base[off0 + off1]);
927 12288 const vec3_t v3 = fn(vec3)(base[off0 + off1 + off2]);
928
929 24576 return (vec3_t) {
930 12288 w0 * v0.x + w1 * v1.x + w2 * v2.x + w3 * v3.x,
931 12288 w0 * v0.y + w1 * v1.y + w2 * v2.y + w3 * v3.y,
932 12288 w0 * v0.z + w1 * v1.z + w2 * v2.z + w3 * v3.z,
933 };
934 }
935
936 av_always_inline static
937 6144 vec3_t fn(lut3d_dynamic)(const SwsLut3D *restrict lut3d, vec3_t rgb)
938 {
939 6144 rgb.x *= (TONE_LUT_SIZE - 1) / (pixel_t) UINT16_MAX;
940
941 /* Linear interpolation */
942 6144 const int Ix = (int) rgb.x;
943 6144 const pixel_t If = rgb.x - Ix;
944
945 6144 const v2u16_t a = lut3d->tone_map[Ix];
946 6144 const v2u16_t b = lut3d->tone_map[Ix + 1];
947
948 6144 const pixel_t k = lerp(a.y, b.y, If);
949 6144 const pixel_t bias = (1 << 15) - k;
950 6144 const pixel_t scale = k / (pixel_t) (1 << 15);
951
952 6144 rgb.x = lerp(a.x, b.x, If);
953 6144 rgb.y = bias + scale * rgb.y;
954 6144 rgb.z = bias + scale * rgb.z;
955
956 /* Re-scale to output LUT size */
957 6144 rgb.x *= (OUTPUT_LUT_SIZE_I - 1) / (pixel_t) UINT16_MAX;
958 6144 rgb.y *= (OUTPUT_LUT_SIZE_PT - 1) / (pixel_t) UINT16_MAX;
959 6144 rgb.z *= (OUTPUT_LUT_SIZE_PT - 1) / (pixel_t) UINT16_MAX;
960
961 /* Trilinear interpolation */
962 6144 const int lo0 = (int) rgb.x;
963 6144 const int lo1 = (int) rgb.y;
964 6144 const int lo2 = (int) rgb.z;
965
966 6144 const int hi0 = FFMIN(lo0 + 1, OUTPUT_LUT_SIZE_I - 1);
967 6144 const int hi1 = FFMIN(lo1 + 1, OUTPUT_LUT_SIZE_PT - 1);
968 6144 const int hi2 = FFMIN(lo2 + 1, OUTPUT_LUT_SIZE_PT - 1);
969
970 6144 const pixel_t w0 = rgb.x - lo0;
971 6144 const vec3_t c000 = fn(vec3)(lut3d->output[lo2][lo1][lo0]);
972 6144 const vec3_t c001 = fn(vec3)(lut3d->output[lo2][lo1][hi0]);
973 6144 const vec3_t c00 = fn(lerp3)(c000, c001, w0);
974 6144 const vec3_t c010 = fn(vec3)(lut3d->output[lo2][hi1][lo0]);
975 6144 const vec3_t c011 = fn(vec3)(lut3d->output[lo2][hi1][hi0]);
976 6144 const vec3_t c01 = fn(lerp3)(c010, c011, w0);
977 6144 const vec3_t c100 = fn(vec3)(lut3d->output[hi2][lo1][lo0]);
978 6144 const vec3_t c101 = fn(vec3)(lut3d->output[hi2][lo1][hi0]);
979 6144 const vec3_t c10 = fn(lerp3)(c100, c101, w0);
980 6144 const vec3_t c110 = fn(vec3)(lut3d->output[hi2][hi1][lo0]);
981 6144 const vec3_t c111 = fn(vec3)(lut3d->output[hi2][hi1][hi0]);
982 6144 const vec3_t c11 = fn(lerp3)(c110, c111, w0);
983
984 6144 const pixel_t w1 = rgb.y - lo1;
985 6144 const vec3_t c0 = fn(lerp3)(c00, c01, w1);
986 6144 const vec3_t c1 = fn(lerp3)(c10, c11, w1);
987
988 6144 const pixel_t w2 = rgb.z - lo2;
989 6144 return fn(lerp3)(c0, c1, w2);
990 }
991
992 384 DECL_FUNC(lut3d, const SwsCompMask mask, const int dynamic)
993 {
994 384 const SwsLut3D *restrict lut3d = impl->priv.ptr;
995
996 SWS_LOOP
997
2/2
✓ Branch 0 taken 12288 times.
✓ Branch 1 taken 384 times.
 12672 for (int i = 0; i < SWS_BLOCK_SIZE; i++) {
998 12288 vec3_t c = { x[i], y[i], z[i] };
999 12288 c = fn(lut3d_static)(lut3d, c);
1000
2/2
✓ Branch 0 taken 6144 times.
✓ Branch 1 taken 6144 times.
 12288 if (dynamic)
1001 6144 c = fn(lut3d_dynamic)(lut3d, c);
1002
1003 12288 x[i] = c.x;
1004 12288 y[i] = c.y;
1005 12288 z[i] = c.z;
1006 }
1007
1008 384 CONTINUE(x, y, z, w);
1009 384 }
1010 #endif /* IS_FLOAT */
1011
1012 768 SWS_FOR(PX, LUT_3D, DECL_IMPL, lut3d)
1013 SWS_FOR_STRUCT(PX, LUT_3D, DECL_ENTRY, .setup = fn(setup_lut3d) )
1014
1015 #undef PIXEL_MAX
1016 #undef PIXEL_SWAP
1017 #undef pixel_t
1018 #undef inter_t
1019 #undef vec3_t
1020 #undef PX
1021 #undef px
1022