FFmpeg coverage

Directory: ../../../ffmpeg/
File: src/libavcodec/x86/vvc/dsp_init.c
Date: 2026-10-30 17:37:53
Exec Total Coverage
Lines: 55 57 96.5%
Functions: 214 214 100.0%
Branches: 18 22 81.8%
Line Branch Exec Source
1 /*
2 * VVC DSP init for x86
3 *
4 * Copyright (C) 2022-2024 Nuo Mi
5 * Copyright (c) 2023-2024 Wu Jianhua
6 *
7 * This file is part of FFmpeg.
8 *
9 * FFmpeg is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * FFmpeg is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with FFmpeg; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 */
23
24 #include "config.h"
25
26 #include "libavutil/attributes.h"
27 #include "libavutil/avassert.h"
28 #include "libavutil/cpu.h"
29 #include "libavutil/x86/cpu.h"
30 #include "libavcodec/vvc/dec.h"
31 #include "libavcodec/vvc/ctu.h"
32 #include "libavcodec/vvc/dsp.h"
33 #include "libavcodec/x86/h26x/h2656dsp.h"
34
35 #if ARCH_X86_64
36
37 #define bf(fn, bd, opt) fn##_##bd##_##opt
38 #define BF(fn, bpc, opt) fn##_##bpc##bpc_##opt
39
40 #define DMVR_PROTOTYPES(bd, opt) \
41 void ff_vvc_dmvr_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
42 int height, intptr_t mx, intptr_t my, int width); \
43 void ff_vvc_dmvr_h_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
44 int height, intptr_t mx, intptr_t my, int width); \
45 void ff_vvc_dmvr_v_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
46 int height, intptr_t mx, intptr_t my, int width); \
47 void ff_vvc_dmvr_hv_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
48 int height, intptr_t mx, intptr_t my, int width); \
49
50 DMVR_PROTOTYPES( 8, avx2)
51 DMVR_PROTOTYPES(10, avx2)
52 DMVR_PROTOTYPES(12, avx2)
53
54 #define OF_INIT(BD, OPT) do { \
55 void ff_vvc_apply_bdof_## BD ## _ ## OPT(uint8_t *dst, ptrdiff_t dst_stride, \
56 const int16_t *src0, const int16_t *src1, \
57 int w, int h); \
58 c->inter.apply_bdof = ff_vvc_apply_bdof_## BD ##_## OPT; \
59 } while (0)
60
61 #define ALF_BPC_PROTOTYPES(bpc, opt) \
62 void BF(ff_vvc_alf_classify_grad, bpc, opt)(int *gradient_sum, \
63 const uint8_t *src, ptrdiff_t src_stride, intptr_t width, intptr_t height, intptr_t vb_pos); \
64 void BF(ff_vvc_alf_classify, bpc, opt)(int *class_idx, int *transpose_idx, const int *gradient_sum, \
65 intptr_t width, intptr_t height, intptr_t vb_pos, intptr_t bit_depth); \
66
67 ALF_BPC_PROTOTYPES(8, avx2)
68 ALF_BPC_PROTOTYPES(16, avx2)
69
70 #if ARCH_X86_64
71 #define FW_PUT(name, depth, opt) \
72 static void vvc_put_ ## name ## _ ## depth ## _##opt(int16_t *dst, const uint8_t *src, ptrdiff_t srcstride, \
73 int height, const int8_t *hf, const int8_t *vf, int width) \
74 { \
75 ff_h2656_put_## name ## _ ## depth ## _##opt(dst, 2 * MAX_PB_SIZE, src, srcstride, height, hf, vf, width); \
76 }
77
78 #if HAVE_SSE4_EXTERNAL
79 #define FW_PUT_TAP(fname, bitd, opt ) \
80 FW_PUT(fname##4, bitd, opt ) \
81 FW_PUT(fname##8, bitd, opt ) \
82 FW_PUT(fname##16, bitd, opt ) \
83 FW_PUT(fname##32, bitd, opt ) \
84 FW_PUT(fname##64, bitd, opt ) \
85 FW_PUT(fname##128, bitd, opt ) \
86
87 #define FW_PUT_4TAP(fname, bitd, opt) \
88 FW_PUT(fname ## 2, bitd, opt) \
89 FW_PUT_TAP(fname, bitd, opt)
90
91 #define FW_PUT_4TAP_SSE4(bitd) \
92 FW_PUT_4TAP(pixels, bitd, sse4) \
93 FW_PUT_4TAP(4tap_h, bitd, sse4) \
94 FW_PUT_4TAP(4tap_v, bitd, sse4) \
95 FW_PUT_4TAP(4tap_hv, bitd, sse4)
96
97 #define FW_PUT_8TAP_SSE4(bitd) \
98 FW_PUT_TAP(8tap_h, bitd, sse4) \
99 FW_PUT_TAP(8tap_v, bitd, sse4) \
100 FW_PUT_TAP(8tap_hv, bitd, sse4)
101
102 #define FW_PUT_SSE4(bitd) \
103 FW_PUT_4TAP_SSE4(bitd) \
104 FW_PUT_8TAP_SSE4(bitd)
105
106 7668 FW_PUT_SSE4( 8)
107 1200 FW_PUT_SSE4(10)
108 1200 FW_PUT_SSE4(12)
109 #endif
110
111 #if HAVE_AVX2_EXTERNAL
112 #define FW_PUT_TAP_AVX2(n, bitd) \
113 FW_PUT(n ## tap_h32, bitd, avx2) \
114 FW_PUT(n ## tap_h64, bitd, avx2) \
115 FW_PUT(n ## tap_h128, bitd, avx2) \
116 FW_PUT(n ## tap_v32, bitd, avx2) \
117 FW_PUT(n ## tap_v64, bitd, avx2) \
118 FW_PUT(n ## tap_v128, bitd, avx2)
119
120 #define FW_PUT_AVX2(bitd) \
121 FW_PUT(pixels32, bitd, avx2) \
122 FW_PUT(pixels64, bitd, avx2) \
123 FW_PUT(pixels128, bitd, avx2) \
124 FW_PUT_TAP_AVX2(4, bitd) \
125 FW_PUT_TAP_AVX2(8, bitd) \
126
127 1840 FW_PUT_AVX2( 8)
128 270 FW_PUT_AVX2(10)
129 270 FW_PUT_AVX2(12)
130
131 #define FW_PUT_TAP_16BPC_AVX2(n, bitd) \
132 FW_PUT(n ## tap_h16, bitd, avx2) \
133 FW_PUT(n ## tap_v16, bitd, avx2) \
134 FW_PUT(n ## tap_hv16, bitd, avx2) \
135 FW_PUT(n ## tap_hv32, bitd, avx2) \
136 FW_PUT(n ## tap_hv64, bitd, avx2) \
137 FW_PUT(n ## tap_hv128, bitd, avx2)
138
139 #define FW_PUT_16BPC_AVX2(bitd) \
140 FW_PUT(pixels16, bitd, avx2) \
141 FW_PUT_TAP_16BPC_AVX2(4, bitd) \
142 FW_PUT_TAP_16BPC_AVX2(8, bitd)
143
144 210 FW_PUT_16BPC_AVX2(10)
145 210 FW_PUT_16BPC_AVX2(12)
146
147 #define ALF_FUNCS(bpc, bd, opt) \
148 static void bf(vvc_alf_classify, bd, opt)(int *class_idx, int *transpose_idx, \
149 const uint8_t *src, ptrdiff_t src_stride, int width, int height, int vb_pos, int *gradient_tmp) \
150 { \
151 BF(ff_vvc_alf_classify_grad, bpc, opt)(gradient_tmp, src, src_stride, width, height, vb_pos); \
152 BF(ff_vvc_alf_classify, bpc, opt)(class_idx, transpose_idx, gradient_tmp, width, height, vb_pos, bd); \
153 } \
154
155 261 ALF_FUNCS(8, 8, avx2)
156 261 ALF_FUNCS(16, 10, avx2)
157 261 ALF_FUNCS(16, 12, avx2)
158
159 #endif
160
161 #define SAO_FILTER_FUNC(wd, bitd, opt) \
162 void ff_vvc_sao_band_filter_##wd##_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
163 const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
164 void ff_vvc_sao_edge_filter_##wd##_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
165 const int16_t *sao_offset_val, int eo, int width, int height); \
166
167 #define SAO_FILTER_FUNCS(bitd, opt) \
168 SAO_FILTER_FUNC(8, bitd, opt) \
169 SAO_FILTER_FUNC(16, bitd, opt) \
170 SAO_FILTER_FUNC(32, bitd, opt) \
171 SAO_FILTER_FUNC(48, bitd, opt) \
172 SAO_FILTER_FUNC(64, bitd, opt) \
173 SAO_FILTER_FUNC(80, bitd, opt) \
174 SAO_FILTER_FUNC(96, bitd, opt) \
175 SAO_FILTER_FUNC(112, bitd, opt) \
176 SAO_FILTER_FUNC(128, bitd, opt) \
177
178 SAO_FILTER_FUNCS(8, avx2)
179 SAO_FILTER_FUNCS(10, avx2)
180 SAO_FILTER_FUNCS(12, avx2)
181
182 #define SAO_FILTER_INIT(type, bitd, opt) do { \
183 c->sao.type##_filter[0] = ff_vvc_sao_##type##_filter_8_##bitd##_##opt; \
184 c->sao.type##_filter[1] = ff_vvc_sao_##type##_filter_16_##bitd##_##opt; \
185 c->sao.type##_filter[2] = ff_vvc_sao_##type##_filter_32_##bitd##_##opt; \
186 c->sao.type##_filter[3] = ff_vvc_sao_##type##_filter_48_##bitd##_##opt; \
187 c->sao.type##_filter[4] = ff_vvc_sao_##type##_filter_64_##bitd##_##opt; \
188 c->sao.type##_filter[5] = ff_vvc_sao_##type##_filter_80_##bitd##_##opt; \
189 c->sao.type##_filter[6] = ff_vvc_sao_##type##_filter_96_##bitd##_##opt; \
190 c->sao.type##_filter[7] = ff_vvc_sao_##type##_filter_112_##bitd##_##opt; \
191 c->sao.type##_filter[8] = ff_vvc_sao_##type##_filter_128_##bitd##_##opt; \
192 } while (0)
193
194 #define SAO_INIT(bitd, opt) do { \
195 SAO_FILTER_INIT(band, bitd, opt); \
196 SAO_FILTER_INIT(edge, bitd, opt); \
197 } while (0)
198
199 #define AVG_INIT(bd, opt) do { \
200 void bf(ff_vvc_avg, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
201 const int16_t *src0, const int16_t *src1, int width, int height);\
202 void bf(ff_vvc_w_avg, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
203 const int16_t *src0, const int16_t *src1, int width, int height, \
204 int denom, int w0, int w1, int o); \
205 c->inter.avg = bf(ff_vvc_avg, bd, opt); \
206 c->inter.w_avg = bf(ff_vvc_w_avg, bd, opt); \
207 } while (0)
208
209 #define DMVR_INIT(bd) do { \
210 c->inter.dmvr[0][0] = ff_vvc_dmvr_##bd##_avx2; \
211 c->inter.dmvr[0][1] = ff_vvc_dmvr_h_##bd##_avx2; \
212 c->inter.dmvr[1][0] = ff_vvc_dmvr_v_##bd##_avx2; \
213 c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_##bd##_avx2; \
214 } while (0)
215
216 #define PEL_LINK(dst, C, W, idx1, idx2, name, D, opt) \
217 dst[C][W][idx1][idx2] = vvc_put_## name ## _ ## D ## _##opt; \
218 dst ## _uni[C][W][idx1][idx2] = ff_h2656_put_uni_ ## name ## _ ## D ## _##opt; \
219
220 #define MC_TAP_LINKS(pointer, C, my, mx, fname, bitd, opt ) \
221 PEL_LINK(pointer, C, 1, my , mx , fname##4 , bitd, opt ); \
222 PEL_LINK(pointer, C, 2, my , mx , fname##8 , bitd, opt ); \
223 PEL_LINK(pointer, C, 3, my , mx , fname##16, bitd, opt ); \
224 PEL_LINK(pointer, C, 4, my , mx , fname##32, bitd, opt ); \
225 PEL_LINK(pointer, C, 5, my , mx , fname##64, bitd, opt ); \
226 PEL_LINK(pointer, C, 6, my , mx , fname##128, bitd, opt );
227
228 #define MC_8TAP_LINKS(pointer, my, mx, fname, bitd, opt) \
229 MC_TAP_LINKS(pointer, LUMA, my, mx, fname, bitd, opt)
230
231 #define MC_8TAP_LINKS_SSE4(bd) \
232 MC_8TAP_LINKS(c->inter.put, 0, 0, pixels, bd, sse4); \
233 MC_8TAP_LINKS(c->inter.put, 0, 1, 8tap_h, bd, sse4); \
234 MC_8TAP_LINKS(c->inter.put, 1, 0, 8tap_v, bd, sse4); \
235 MC_8TAP_LINKS(c->inter.put, 1, 1, 8tap_hv, bd, sse4)
236
237 #define MC_4TAP_LINKS(pointer, my, mx, fname, bitd, opt) \
238 PEL_LINK(pointer, CHROMA, 0, my , mx , fname##2 , bitd, opt ); \
239 MC_TAP_LINKS(pointer, CHROMA, my, mx, fname, bitd, opt) \
240
241 #define MC_4TAP_LINKS_SSE4(bd) \
242 MC_4TAP_LINKS(c->inter.put, 0, 0, pixels, bd, sse4); \
243 MC_4TAP_LINKS(c->inter.put, 0, 1, 4tap_h, bd, sse4); \
244 MC_4TAP_LINKS(c->inter.put, 1, 0, 4tap_v, bd, sse4); \
245 MC_4TAP_LINKS(c->inter.put, 1, 1, 4tap_hv, bd, sse4)
246
247 #define MC_LINK_SSE4(bd) \
248 MC_4TAP_LINKS_SSE4(bd) \
249 MC_8TAP_LINKS_SSE4(bd)
250
251 #define MC_TAP_LINKS_AVX2(C,tap,bd) do { \
252 PEL_LINK(c->inter.put, C, 4, 0, 0, pixels32, bd, avx2) \
253 PEL_LINK(c->inter.put, C, 5, 0, 0, pixels64, bd, avx2) \
254 PEL_LINK(c->inter.put, C, 6, 0, 0, pixels128, bd, avx2) \
255 PEL_LINK(c->inter.put, C, 4, 0, 1, tap##tap_h32, bd, avx2) \
256 PEL_LINK(c->inter.put, C, 5, 0, 1, tap##tap_h64, bd, avx2) \
257 PEL_LINK(c->inter.put, C, 6, 0, 1, tap##tap_h128, bd, avx2) \
258 PEL_LINK(c->inter.put, C, 4, 1, 0, tap##tap_v32, bd, avx2) \
259 PEL_LINK(c->inter.put, C, 5, 1, 0, tap##tap_v64, bd, avx2) \
260 PEL_LINK(c->inter.put, C, 6, 1, 0, tap##tap_v128, bd, avx2) \
261 } while (0)
262
263 #define MC_LINKS_AVX2(bd) \
264 MC_TAP_LINKS_AVX2(LUMA, 8, bd); \
265 MC_TAP_LINKS_AVX2(CHROMA, 4, bd);
266
267 #define MC_TAP_LINKS_16BPC_AVX2(C, tap, bd) do { \
268 PEL_LINK(c->inter.put, C, 3, 0, 0, pixels16, bd, avx2) \
269 PEL_LINK(c->inter.put, C, 3, 0, 1, tap##tap_h16, bd, avx2) \
270 PEL_LINK(c->inter.put, C, 3, 1, 0, tap##tap_v16, bd, avx2) \
271 PEL_LINK(c->inter.put, C, 3, 1, 1, tap##tap_hv16, bd, avx2) \
272 PEL_LINK(c->inter.put, C, 4, 1, 1, tap##tap_hv32, bd, avx2) \
273 PEL_LINK(c->inter.put, C, 5, 1, 1, tap##tap_hv64, bd, avx2) \
274 PEL_LINK(c->inter.put, C, 6, 1, 1, tap##tap_hv128, bd, avx2) \
275 } while (0)
276
277 #define MC_LINKS_16BPC_AVX2(bd) \
278 MC_TAP_LINKS_16BPC_AVX2(LUMA, 8, bd); \
279 MC_TAP_LINKS_16BPC_AVX2(CHROMA, 4, bd);
280
281 int ff_vvc_sad_avx2(const int16_t *src0, const int16_t *src1, int dx, int dy, int block_w, int block_h);
282 #define SAD_INIT() c->inter.sad = ff_vvc_sad_avx2
283
284 #define ALF_INIT(bd, opt) do { \
285 void bf(ff_vvc_alf_filter_luma, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
286 const uint8_t *src, ptrdiff_t src_stride, int width, int height, \
287 const int16_t *filter, const int16_t *clip, int vb_pos); \
288 void bf(ff_vvc_alf_filter_chroma, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
289 const uint8_t *src, ptrdiff_t src_stride, int width, int height, \
290 const int16_t *filter, const int16_t *clip, int vb_pos); \
291 c->alf.filter[LUMA] = bf(ff_vvc_alf_filter_luma, bd, opt); \
292 c->alf.filter[CHROMA] = bf(ff_vvc_alf_filter_chroma, bd, opt); \
293 c->alf.classify = bf(vvc_alf_classify, bd, opt); \
294 } while (0)
295
296 #endif
297
298 #if HAVE_AVX2_EXTERNAL
299 #define PLANAR_PROTOTYPE(wd, bd, opt) \
300 void ff_vvc_pred_planar_w##wd##_##bd##_##opt(uint8_t *src, const uint8_t *top, \
301 const uint8_t *left, int w, int h, ptrdiff_t stride);
302
303 #define PLANAR_PROTOTYPES(bd, opt) \
304 PLANAR_PROTOTYPE( 4, bd, opt) \
305 PLANAR_PROTOTYPE( 8, bd, opt) \
306 PLANAR_PROTOTYPE(16, bd, opt) \
307 PLANAR_PROTOTYPE(32, bd, opt) \
308 PLANAR_PROTOTYPE(64, bd, opt)
309
310 PLANAR_PROTOTYPES(8, avx2)
311
312 /* pred_planar is one pointer taking a runtime width, so dispatch here. */
313 #define PLANAR_DISPATCH(bd, opt) \
314 static void vvc_pred_planar_##bd##_##opt(uint8_t *src, const uint8_t *top, \
315 const uint8_t *left, const int w, const int h, const ptrdiff_t stride) \
316 { \
317 static void (*const planar[])(uint8_t *src, const uint8_t *top, \
318 const uint8_t *left, int w, int h, ptrdiff_t stride) = { \
319 ff_vvc_pred_planar_w4_##bd##_##opt, \
320 ff_vvc_pred_planar_w8_##bd##_##opt, \
321 ff_vvc_pred_planar_w16_##bd##_##opt, \
322 ff_vvc_pred_planar_w32_##bd##_##opt, \
323 ff_vvc_pred_planar_w64_##bd##_##opt, \
324 }; \
325 av_assert2(w >= 4 && w <= 64 && !(w & (w - 1))); \
326 planar[av_log2(w) - 2](src, top, left, w, h, stride); \
327 }
328
329 355 PLANAR_DISPATCH(8, avx2)
330
331 #define INTRA_INIT(bd, opt) do { \
332 c->intra.pred_planar = vvc_pred_planar_##bd##_##opt; \
333 } while (0)
334 #endif
335
336
337 #endif // ARCH_X86_64
338
339 1654 av_cold void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
340 {
341 #if ARCH_X86_64
342 1654 const int cpu_flags = av_get_cpu_flags();
343
344
3/4
✓ Branch 0 taken 271 times.
✓ Branch 1 taken 1215 times.
✓ Branch 2 taken 168 times.
✗ Branch 3 not taken.
 1654 switch (bd) {
345 271 case 8:
346 #if HAVE_SSE4_EXTERNAL
347
2/2
✓ Branch 0 taken 116 times.
✓ Branch 1 taken 155 times.
 271 if (EXTERNAL_SSE4(cpu_flags)) {
348 116 MC_LINK_SSE4(8);
349 }
350 #endif
351 #if HAVE_AVX2_EXTERNAL
352
3/4
✓ Branch 0 taken 44 times.
✓ Branch 1 taken 227 times.
✓ Branch 2 taken 44 times.
✗ Branch 3 not taken.
 271 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
353 // inter
354 44 AVG_INIT(8, avx2);
355 44 DMVR_INIT(8);
356 44 MC_LINKS_AVX2(8);
357 44 OF_INIT(8, avx2);
358 44 SAD_INIT();
359
360 // intra
361 44 INTRA_INIT(8, avx2);
362
363 // filter
364 44 ALF_INIT(8, avx2);
365 44 SAO_INIT(8, avx2);
366 }
367 #endif
368 271 break;
369 1215 case 10:
370 #if HAVE_SSE4_EXTERNAL
371
2/2
✓ Branch 0 taken 91 times.
✓ Branch 1 taken 1124 times.
 1215 if (EXTERNAL_SSE4(cpu_flags)) {
372 91 MC_LINK_SSE4(10);
373 }
374 #endif
375 #if HAVE_AVX2_EXTERNAL
376
3/4
✓ Branch 0 taken 13 times.
✓ Branch 1 taken 1202 times.
✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
 1215 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
377 // inter
378 13 AVG_INIT(10, avx2);
379 13 DMVR_INIT(10);
380 13 MC_LINKS_AVX2(10);
381 13 MC_LINKS_16BPC_AVX2(10);
382 13 OF_INIT(10, avx2);
383 13 SAD_INIT();
384
385 // filter
386 13 ALF_INIT(10, avx2);
387 13 SAO_INIT(10, avx2);
388 }
389 #endif
390 1215 break;
391 168 case 12:
392 #if HAVE_SSE4_EXTERNAL
393
2/2
✓ Branch 0 taken 84 times.
✓ Branch 1 taken 84 times.
 168 if (EXTERNAL_SSE4(cpu_flags)) {
394 84 MC_LINK_SSE4(12);
395 }
396 #endif
397 #if HAVE_AVX2_EXTERNAL
398
3/4
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 156 times.
✓ Branch 2 taken 12 times.
✗ Branch 3 not taken.
 168 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
399 // inter
400 12 AVG_INIT(12, avx2);
401 12 DMVR_INIT(12);
402 12 MC_LINKS_AVX2(12);
403 12 MC_LINKS_16BPC_AVX2(12);
404 12 OF_INIT(12, avx2);
405 12 SAD_INIT();
406
407 // filter
408 12 ALF_INIT(12, avx2);
409 12 SAO_INIT(12, avx2);
410 }
411 #endif
412 168 break;
413 ✗ default:
414 ✗ break;
415 }
416 #endif
417 1654 }
418