LCOV - code coverage report
Current view: top level - src/libavcodec/x86 - hevcdsp_init.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 57 583 9.8 %
Date: 2017-01-22 02:20:28 Functions: 1 860 0.1 %

          Line data    Source code
       1             : /*
       2             :  * Copyright (c) 2013 Seppo Tomperi
       3             :  * Copyright (c) 2013 - 2014 Pierre-Edouard Lepere
       4             :  *
       5             :  *
       6             :  * This file is part of FFmpeg.
       7             :  *
       8             :  * FFmpeg is free software; you can redistribute it and/or
       9             :  * modify it under the terms of the GNU Lesser General Public
      10             :  * License as published by the Free Software Foundation; either
      11             :  * version 2.1 of the License, or (at your option) any later version.
      12             :  *
      13             :  * FFmpeg is distributed in the hope that it will be useful,
      14             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      15             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      16             :  * Lesser General Public License for more details.
      17             :  *
      18             :  * You should have received a copy of the GNU Lesser General Public
      19             :  * License along with FFmpeg; if not, write to the Free Software
      20             :  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
      21             :  */
      22             : 
      23             : #include "config.h"
      24             : #include "libavutil/cpu.h"
      25             : #include "libavutil/x86/asm.h"
      26             : #include "libavutil/x86/cpu.h"
      27             : #include "libavcodec/get_bits.h" /* required for hevcdsp.h GetBitContext */
      28             : #include "libavcodec/hevcdsp.h"
      29             : #include "libavcodec/x86/hevcdsp.h"
      30             : 
      31             : #define LFC_FUNC(DIR, DEPTH, OPT) \
      32             : void ff_hevc_ ## DIR ## _loop_filter_chroma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int *tc, uint8_t *no_p, uint8_t *no_q);
      33             : 
      34             : #define LFL_FUNC(DIR, DEPTH, OPT) \
      35             : void ff_hevc_ ## DIR ## _loop_filter_luma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int beta, int *tc, uint8_t *no_p, uint8_t *no_q);
      36             : 
      37             : #define LFC_FUNCS(type, depth, opt) \
      38             :     LFC_FUNC(h, depth, opt)  \
      39             :     LFC_FUNC(v, depth, opt)
      40             : 
      41             : #define LFL_FUNCS(type, depth, opt) \
      42             :     LFL_FUNC(h, depth, opt)  \
      43             :     LFL_FUNC(v, depth, opt)
      44             : 
      45             : LFC_FUNCS(uint8_t,   8, sse2)
      46             : LFC_FUNCS(uint8_t,  10, sse2)
      47             : LFC_FUNCS(uint8_t,  12, sse2)
      48             : LFC_FUNCS(uint8_t,   8, avx)
      49             : LFC_FUNCS(uint8_t,  10, avx)
      50             : LFC_FUNCS(uint8_t,  12, avx)
      51             : LFL_FUNCS(uint8_t,   8, sse2)
      52             : LFL_FUNCS(uint8_t,  10, sse2)
      53             : LFL_FUNCS(uint8_t,  12, sse2)
      54             : LFL_FUNCS(uint8_t,   8, ssse3)
      55             : LFL_FUNCS(uint8_t,  10, ssse3)
      56             : LFL_FUNCS(uint8_t,  12, ssse3)
      57             : LFL_FUNCS(uint8_t,   8, avx)
      58             : LFL_FUNCS(uint8_t,  10, avx)
      59             : LFL_FUNCS(uint8_t,  12, avx)
      60             : 
      61             : #define IDCT_FUNCS(W, opt) \
      62             : void ff_hevc_idct##W##_dc_8_##opt(int16_t *coeffs); \
      63             : void ff_hevc_idct##W##_dc_10_##opt(int16_t *coeffs); \
      64             : void ff_hevc_idct##W##_dc_12_##opt(int16_t *coeffs)
      65             : 
      66             : IDCT_FUNCS(4x4,   mmxext);
      67             : IDCT_FUNCS(8x8,   mmxext);
      68             : IDCT_FUNCS(8x8,   sse2);
      69             : IDCT_FUNCS(16x16, sse2);
      70             : IDCT_FUNCS(32x32, sse2);
      71             : IDCT_FUNCS(16x16, avx2);
      72             : IDCT_FUNCS(32x32, avx2);
      73             : 
      74             : #define mc_rep_func(name, bitd, step, W, opt) \
      75             : void ff_hevc_put_hevc_##name##W##_##bitd##_##opt(int16_t *_dst,                                                 \
      76             :                                                 uint8_t *_src, ptrdiff_t _srcstride, int height,                \
      77             :                                                 intptr_t mx, intptr_t my, int width)                            \
      78             : {                                                                                                               \
      79             :     int i;                                                                                                      \
      80             :     uint8_t *src;                                                                                               \
      81             :     int16_t *dst;                                                                                               \
      82             :     for (i = 0; i < W; i += step) {                                                                             \
      83             :         src  = _src + (i * ((bitd + 7) / 8));                                                                   \
      84             :         dst = _dst + i;                                                                                         \
      85             :         ff_hevc_put_hevc_##name##step##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width);            \
      86             :     }                                                                                                           \
      87             : }
      88             : #define mc_rep_uni_func(name, bitd, step, W, opt) \
      89             : void ff_hevc_put_hevc_uni_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride,                        \
      90             :                                                     uint8_t *_src, ptrdiff_t _srcstride, int height,            \
      91             :                                                     intptr_t mx, intptr_t my, int width)                        \
      92             : {                                                                                                               \
      93             :     int i;                                                                                                      \
      94             :     uint8_t *src;                                                                                               \
      95             :     uint8_t *dst;                                                                                               \
      96             :     for (i = 0; i < W; i += step) {                                                                             \
      97             :         src = _src + (i * ((bitd + 7) / 8));                                                                    \
      98             :         dst = _dst + (i * ((bitd + 7) / 8));                                                                    \
      99             :         ff_hevc_put_hevc_uni_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride,                     \
     100             :                                                           height, mx, my, width);                               \
     101             :     }                                                                                                           \
     102             : }
     103             : #define mc_rep_bi_func(name, bitd, step, W, opt) \
     104             : void ff_hevc_put_hevc_bi_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, uint8_t *_src,          \
     105             :                                                    ptrdiff_t _srcstride, int16_t* _src2,                        \
     106             :                                                    int height, intptr_t mx, intptr_t my, int width)             \
     107             : {                                                                                                               \
     108             :     int i;                                                                                                      \
     109             :     uint8_t  *src;                                                                                              \
     110             :     uint8_t  *dst;                                                                                              \
     111             :     int16_t  *src2;                                                                                             \
     112             :     for (i = 0; i < W ; i += step) {                                                                            \
     113             :         src  = _src + (i * ((bitd + 7) / 8));                                                                   \
     114             :         dst  = _dst + (i * ((bitd + 7) / 8));                                                                   \
     115             :         src2 = _src2 + i;                                                                                       \
     116             :         ff_hevc_put_hevc_bi_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2,                \
     117             :                                                           height, mx, my, width);                               \
     118             :     }                                                                                                           \
     119             : }
     120             : 
     121             : #define mc_rep_funcs(name, bitd, step, W, opt)        \
     122             :     mc_rep_func(name, bitd, step, W, opt)            \
     123             :     mc_rep_uni_func(name, bitd, step, W, opt)        \
     124             :     mc_rep_bi_func(name, bitd, step, W, opt)
     125             : 
     126             : #define mc_rep_func2(name, bitd, step1, step2, W, opt) \
     127             : void ff_hevc_put_hevc_##name##W##_##bitd##_##opt(int16_t *dst,                                                  \
     128             :                                                  uint8_t *src, ptrdiff_t _srcstride, int height,                \
     129             :                                                  intptr_t mx, intptr_t my, int width)                           \
     130             : {                                                                                                               \
     131             :     ff_hevc_put_hevc_##name##step1##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width);               \
     132             :     ff_hevc_put_hevc_##name##step2##_##bitd##_##opt(dst + step1, src + (step1 * ((bitd + 7) / 8)),              \
     133             :                                                     _srcstride, height, mx, my, width);                         \
     134             : }
     135             : #define mc_rep_uni_func2(name, bitd, step1, step2, W, opt) \
     136             : void ff_hevc_put_hevc_uni_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride,                         \
     137             :                                                      uint8_t *src, ptrdiff_t _srcstride, int height,            \
     138             :                                                      intptr_t mx, intptr_t my, int width)                       \
     139             : {                                                                                                               \
     140             :     ff_hevc_put_hevc_uni_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, height, mx, my, width);\
     141             :     ff_hevc_put_hevc_uni_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride,            \
     142             :                                                         src + (step1 * ((bitd + 7) / 8)), _srcstride,           \
     143             :                                                         height, mx, my, width);                                 \
     144             : }
     145             : #define mc_rep_bi_func2(name, bitd, step1, step2, W, opt) \
     146             : void ff_hevc_put_hevc_bi_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, uint8_t *src,            \
     147             :                                                     ptrdiff_t _srcstride, int16_t* src2,                        \
     148             :                                                     int height, intptr_t mx, intptr_t my, int width)            \
     149             : {                                                                                                               \
     150             :     ff_hevc_put_hevc_bi_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, height, mx, my, width);\
     151             :     ff_hevc_put_hevc_bi_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride,             \
     152             :                                                        src + (step1 * ((bitd + 7) / 8)), _srcstride,            \
     153             :                                                        src2 + step1, height, mx, my, width);                    \
     154             : }
     155             : 
     156             : #define mc_rep_funcs2(name, bitd, step1, step2, W, opt) \
     157             :     mc_rep_func2(name, bitd, step1, step2, W, opt)      \
     158             :     mc_rep_uni_func2(name, bitd, step1, step2, W, opt)  \
     159             :     mc_rep_bi_func2(name, bitd, step1, step2, W, opt)
     160             : 
     161             : #if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
     162             : 
     163             : #define mc_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)                                       \
     164             : void ff_hevc_put_hevc_##name##width1##_10_##opt1(int16_t *dst, uint8_t *src, ptrdiff_t _srcstride,            \
     165             :                                                  int height, intptr_t mx, intptr_t my, int width)             \
     166             :                                                                                                               \
     167             : {                                                                                                             \
     168             :     ff_hevc_put_hevc_##name##width2##_10_##opt1(dst, src, _srcstride, height, mx, my, width);                 \
     169             :     ff_hevc_put_hevc_##name##width3##_10_##opt2(dst+ width2, src+ width4, _srcstride, height, mx, my, width); \
     170             : }
     171             : 
     172             : #define mc_bi_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)                                    \
     173             : void ff_hevc_put_hevc_bi_##name##width1##_10_##opt1(uint8_t *dst, ptrdiff_t dststride, uint8_t *src,          \
     174             :                                                     ptrdiff_t _srcstride, int16_t *src2,                      \
     175             :                                                     int height, intptr_t mx, intptr_t my, int width)          \
     176             : {                                                                                                             \
     177             :     ff_hevc_put_hevc_bi_##name##width2##_10_##opt1(dst, dststride, src, _srcstride, src2,                     \
     178             :                                                    height, mx, my, width);                                    \
     179             :     ff_hevc_put_hevc_bi_##name##width3##_10_##opt2(dst+width4, dststride, src+width4, _srcstride, src2+width2,\
     180             :                                                    height, mx, my, width);                                    \
     181             : }
     182             : 
     183             : #define mc_uni_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)                                   \
     184             : void ff_hevc_put_hevc_uni_##name##width1##_10_##opt1(uint8_t *dst, ptrdiff_t dststride,                       \
     185             :                                                      uint8_t *src, ptrdiff_t _srcstride, int height,          \
     186             :                                                      intptr_t mx, intptr_t my, int width)                     \
     187             : {                                                                                                             \
     188             :     ff_hevc_put_hevc_uni_##name##width2##_10_##opt1(dst, dststride, src, _srcstride,                          \
     189             :                                                       height, mx, my, width);                                 \
     190             :     ff_hevc_put_hevc_uni_##name##width3##_10_##opt2(dst+width4, dststride, src+width4, _srcstride,            \
     191             :                                                       height, mx, my, width);                                 \
     192             : }
     193             : 
     194             : #define mc_rep_mixs_10(name, width1, width2, width3, opt1, opt2, width4)   \
     195             : mc_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)            \
     196             : mc_bi_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)         \
     197             : mc_uni_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)
     198             : 
     199             : #define mc_rep_mix_8(name, width1, width2, width3, opt1, opt2)                                                \
     200             : void ff_hevc_put_hevc_##name##width1##_8_##opt1(int16_t *dst, uint8_t *src, ptrdiff_t _srcstride,             \
     201             :                                                 int height, intptr_t mx, intptr_t my, int width)              \
     202             :                                                                                                               \
     203             : {                                                                                                             \
     204             :     ff_hevc_put_hevc_##name##width2##_8_##opt1(dst, src, _srcstride, height, mx, my, width);                  \
     205             :     ff_hevc_put_hevc_##name##width3##_8_##opt2(dst+ width2, src+ width2, _srcstride, height, mx, my, width);  \
     206             : }
     207             : 
     208             : #define mc_bi_rep_mix_8(name, width1, width2, width3, opt1, opt2)                                             \
     209             : void ff_hevc_put_hevc_bi_##name##width1##_8_##opt1(uint8_t *dst, ptrdiff_t dststride, uint8_t *src,           \
     210             :                                                    ptrdiff_t _srcstride, int16_t* src2,                       \
     211             :                                                    int height, intptr_t mx, intptr_t my, int width)           \
     212             : {                                                                                                             \
     213             :     ff_hevc_put_hevc_bi_##name##width2##_8_##opt1(dst, dststride, src, _srcstride,                            \
     214             :                                                   src2, height, mx, my, width);                               \
     215             :     ff_hevc_put_hevc_bi_##name##width3##_8_##opt2(dst+width2, dststride, src+width2, _srcstride,              \
     216             :                                                   src2+width2, height, mx, my, width);                        \
     217             : }
     218             : 
     219             : #define mc_uni_rep_mix_8(name, width1, width2, width3, opt1, opt2)                                            \
     220             : void ff_hevc_put_hevc_uni_##name##width1##_8_##opt1(uint8_t *dst, ptrdiff_t dststride,                        \
     221             :                                                     uint8_t *src, ptrdiff_t _srcstride, int height,           \
     222             :                                                     intptr_t mx, intptr_t my, int width)                      \
     223             : {                                                                                                             \
     224             :     ff_hevc_put_hevc_uni_##name##width2##_8_##opt1(dst, dststride, src, _srcstride,                           \
     225             :                                                    height, mx, my, width);                                    \
     226             :     ff_hevc_put_hevc_uni_##name##width3##_8_##opt2(dst+width2, dststride, src+width2, _srcstride,             \
     227             :                                                    height, mx, my, width);                                    \
     228             : }
     229             : 
     230             : #define mc_rep_mixs_8(name, width1, width2, width3, opt1, opt2)   \
     231             : mc_rep_mix_8(name, width1, width2, width3, opt1, opt2)            \
     232             : mc_bi_rep_mix_8(name, width1, width2, width3, opt1, opt2)         \
     233             : mc_uni_rep_mix_8(name, width1, width2, width3, opt1, opt2)
     234             : 
     235             : #if HAVE_AVX2_EXTERNAL
     236             : 
     237           0 : mc_rep_mixs_8(pel_pixels, 48, 32, 16, avx2, sse4)
     238           0 : mc_rep_mixs_8(epel_hv,    48, 32, 16, avx2, sse4)
     239           0 : mc_rep_mixs_8(epel_h ,    48, 32, 16, avx2, sse4)
     240           0 : mc_rep_mixs_8(epel_v ,    48, 32, 16, avx2, sse4)
     241             : 
     242           0 : mc_rep_mix_10(pel_pixels, 24, 16, 8, avx2, sse4, 32)
     243           0 : mc_bi_rep_mix_10(pel_pixels,24, 16, 8, avx2, sse4, 32)
     244           0 : mc_rep_mixs_10(epel_hv,   24, 16, 8, avx2, sse4, 32)
     245           0 : mc_rep_mixs_10(epel_h ,   24, 16, 8, avx2, sse4, 32)
     246           0 : mc_rep_mixs_10(epel_v ,   24, 16, 8, avx2, sse4, 32)
     247             : 
     248             : 
     249           0 : mc_rep_mixs_10(qpel_h ,   24, 16, 8, avx2, sse4, 32)
     250           0 : mc_rep_mixs_10(qpel_v ,   24, 16, 8, avx2, sse4, 32)
     251           0 : mc_rep_mixs_10(qpel_hv,   24, 16, 8, avx2, sse4, 32)
     252             : 
     253             : 
     254           0 : mc_rep_uni_func(pel_pixels, 8, 64, 128, avx2)//used for 10bit
     255           0 : mc_rep_uni_func(pel_pixels, 8, 32, 96, avx2) //used for 10bit
     256             : 
     257           0 : mc_rep_funcs(pel_pixels, 8, 32, 64, avx2)
     258             : 
     259           0 : mc_rep_func(pel_pixels, 10, 16, 32, avx2)
     260           0 : mc_rep_func(pel_pixels, 10, 16, 48, avx2)
     261           0 : mc_rep_func(pel_pixels, 10, 32, 64, avx2)
     262             : 
     263           0 : mc_rep_bi_func(pel_pixels, 10, 16, 32, avx2)
     264           0 : mc_rep_bi_func(pel_pixels, 10, 16, 48, avx2)
     265           0 : mc_rep_bi_func(pel_pixels, 10, 32, 64, avx2)
     266             : 
     267           0 : mc_rep_funcs(epel_h, 8, 32, 64, avx2)
     268             : 
     269           0 : mc_rep_funcs(epel_v, 8, 32, 64, avx2)
     270             : 
     271           0 : mc_rep_funcs(epel_h, 10, 16, 32, avx2)
     272           0 : mc_rep_funcs(epel_h, 10, 16, 48, avx2)
     273           0 : mc_rep_funcs(epel_h, 10, 32, 64, avx2)
     274             : 
     275           0 : mc_rep_funcs(epel_v, 10, 16, 32, avx2)
     276           0 : mc_rep_funcs(epel_v, 10, 16, 48, avx2)
     277           0 : mc_rep_funcs(epel_v, 10, 32, 64, avx2)
     278             : 
     279             : 
     280           0 : mc_rep_funcs(epel_hv,  8, 32, 64, avx2)
     281             : 
     282           0 : mc_rep_funcs(epel_hv, 10, 16, 32, avx2)
     283           0 : mc_rep_funcs(epel_hv, 10, 16, 48, avx2)
     284           0 : mc_rep_funcs(epel_hv, 10, 32, 64, avx2)
     285             : 
     286           0 : mc_rep_funcs(qpel_h, 8, 32, 64, avx2)
     287           0 : mc_rep_mixs_8(qpel_h ,  48, 32, 16, avx2, sse4)
     288             : 
     289           0 : mc_rep_funcs(qpel_v, 8, 32, 64, avx2)
     290           0 : mc_rep_mixs_8(qpel_v,  48, 32, 16, avx2, sse4)
     291             : 
     292           0 : mc_rep_funcs(qpel_h, 10, 16, 32, avx2)
     293           0 : mc_rep_funcs(qpel_h, 10, 16, 48, avx2)
     294           0 : mc_rep_funcs(qpel_h, 10, 32, 64, avx2)
     295             : 
     296           0 : mc_rep_funcs(qpel_v, 10, 16, 32, avx2)
     297           0 : mc_rep_funcs(qpel_v, 10, 16, 48, avx2)
     298           0 : mc_rep_funcs(qpel_v, 10, 32, 64, avx2)
     299             : 
     300           0 : mc_rep_funcs(qpel_hv, 10, 16, 32, avx2)
     301           0 : mc_rep_funcs(qpel_hv, 10, 16, 48, avx2)
     302           0 : mc_rep_funcs(qpel_hv, 10, 32, 64, avx2)
     303             : 
     304             : #endif //AVX2
     305             : 
     306           0 : mc_rep_funcs(pel_pixels, 8, 16, 64, sse4)
     307           0 : mc_rep_funcs(pel_pixels, 8, 16, 48, sse4)
     308           0 : mc_rep_funcs(pel_pixels, 8, 16, 32, sse4)
     309           0 : mc_rep_funcs(pel_pixels, 8,  8, 24, sse4)
     310           0 : mc_rep_funcs(pel_pixels,10,  8, 64, sse4)
     311           0 : mc_rep_funcs(pel_pixels,10,  8, 48, sse4)
     312           0 : mc_rep_funcs(pel_pixels,10,  8, 32, sse4)
     313           0 : mc_rep_funcs(pel_pixels,10,  8, 24, sse4)
     314           0 : mc_rep_funcs(pel_pixels,10,  8, 16, sse4)
     315           0 : mc_rep_funcs(pel_pixels,10,  4, 12, sse4)
     316           0 : mc_rep_funcs(pel_pixels,12,  8, 64, sse4)
     317           0 : mc_rep_funcs(pel_pixels,12,  8, 48, sse4)
     318           0 : mc_rep_funcs(pel_pixels,12,  8, 32, sse4)
     319           0 : mc_rep_funcs(pel_pixels,12,  8, 24, sse4)
     320           0 : mc_rep_funcs(pel_pixels,12,  8, 16, sse4)
     321           0 : mc_rep_funcs(pel_pixels,12,  4, 12, sse4)
     322             : 
     323           0 : mc_rep_funcs(epel_h, 8, 16, 64, sse4)
     324           0 : mc_rep_funcs(epel_h, 8, 16, 48, sse4)
     325           0 : mc_rep_funcs(epel_h, 8, 16, 32, sse4)
     326           0 : mc_rep_funcs(epel_h, 8,  8, 24, sse4)
     327           0 : mc_rep_funcs(epel_h,10,  8, 64, sse4)
     328           0 : mc_rep_funcs(epel_h,10,  8, 48, sse4)
     329           0 : mc_rep_funcs(epel_h,10,  8, 32, sse4)
     330           0 : mc_rep_funcs(epel_h,10,  8, 24, sse4)
     331           0 : mc_rep_funcs(epel_h,10,  8, 16, sse4)
     332           0 : mc_rep_funcs(epel_h,10,  4, 12, sse4)
     333           0 : mc_rep_funcs(epel_h,12,  8, 64, sse4)
     334           0 : mc_rep_funcs(epel_h,12,  8, 48, sse4)
     335           0 : mc_rep_funcs(epel_h,12,  8, 32, sse4)
     336           0 : mc_rep_funcs(epel_h,12,  8, 24, sse4)
     337           0 : mc_rep_funcs(epel_h,12,  8, 16, sse4)
     338           0 : mc_rep_funcs(epel_h,12,  4, 12, sse4)
     339           0 : mc_rep_funcs(epel_v, 8, 16, 64, sse4)
     340           0 : mc_rep_funcs(epel_v, 8, 16, 48, sse4)
     341           0 : mc_rep_funcs(epel_v, 8, 16, 32, sse4)
     342           0 : mc_rep_funcs(epel_v, 8,  8, 24, sse4)
     343           0 : mc_rep_funcs(epel_v,10,  8, 64, sse4)
     344           0 : mc_rep_funcs(epel_v,10,  8, 48, sse4)
     345           0 : mc_rep_funcs(epel_v,10,  8, 32, sse4)
     346           0 : mc_rep_funcs(epel_v,10,  8, 24, sse4)
     347           0 : mc_rep_funcs(epel_v,10,  8, 16, sse4)
     348           0 : mc_rep_funcs(epel_v,10,  4, 12, sse4)
     349           0 : mc_rep_funcs(epel_v,12,  8, 64, sse4)
     350           0 : mc_rep_funcs(epel_v,12,  8, 48, sse4)
     351           0 : mc_rep_funcs(epel_v,12,  8, 32, sse4)
     352           0 : mc_rep_funcs(epel_v,12,  8, 24, sse4)
     353           0 : mc_rep_funcs(epel_v,12,  8, 16, sse4)
     354           0 : mc_rep_funcs(epel_v,12,  4, 12, sse4)
     355           0 : mc_rep_funcs(epel_hv, 8, 16, 64, sse4)
     356           0 : mc_rep_funcs(epel_hv, 8, 16, 48, sse4)
     357           0 : mc_rep_funcs(epel_hv, 8, 16, 32, sse4)
     358           0 : mc_rep_funcs(epel_hv, 8,  8, 24, sse4)
     359           0 : mc_rep_funcs2(epel_hv,8,  8,  4, 12, sse4)
     360           0 : mc_rep_funcs(epel_hv,10,  8, 64, sse4)
     361           0 : mc_rep_funcs(epel_hv,10,  8, 48, sse4)
     362           0 : mc_rep_funcs(epel_hv,10,  8, 32, sse4)
     363           0 : mc_rep_funcs(epel_hv,10,  8, 24, sse4)
     364           0 : mc_rep_funcs(epel_hv,10,  8, 16, sse4)
     365           0 : mc_rep_funcs(epel_hv,10,  4, 12, sse4)
     366           0 : mc_rep_funcs(epel_hv,12,  8, 64, sse4)
     367           0 : mc_rep_funcs(epel_hv,12,  8, 48, sse4)
     368           0 : mc_rep_funcs(epel_hv,12,  8, 32, sse4)
     369           0 : mc_rep_funcs(epel_hv,12,  8, 24, sse4)
     370           0 : mc_rep_funcs(epel_hv,12,  8, 16, sse4)
     371           0 : mc_rep_funcs(epel_hv,12,  4, 12, sse4)
     372             : 
     373           0 : mc_rep_funcs(qpel_h, 8, 16, 64, sse4)
     374           0 : mc_rep_funcs(qpel_h, 8, 16, 48, sse4)
     375           0 : mc_rep_funcs(qpel_h, 8, 16, 32, sse4)
     376           0 : mc_rep_funcs(qpel_h, 8,  8, 24, sse4)
     377           0 : mc_rep_funcs(qpel_h,10,  8, 64, sse4)
     378           0 : mc_rep_funcs(qpel_h,10,  8, 48, sse4)
     379           0 : mc_rep_funcs(qpel_h,10,  8, 32, sse4)
     380           0 : mc_rep_funcs(qpel_h,10,  8, 24, sse4)
     381           0 : mc_rep_funcs(qpel_h,10,  8, 16, sse4)
     382           0 : mc_rep_funcs(qpel_h,10,  4, 12, sse4)
     383           0 : mc_rep_funcs(qpel_h,12,  8, 64, sse4)
     384           0 : mc_rep_funcs(qpel_h,12,  8, 48, sse4)
     385           0 : mc_rep_funcs(qpel_h,12,  8, 32, sse4)
     386           0 : mc_rep_funcs(qpel_h,12,  8, 24, sse4)
     387           0 : mc_rep_funcs(qpel_h,12,  8, 16, sse4)
     388           0 : mc_rep_funcs(qpel_h,12,  4, 12, sse4)
     389           0 : mc_rep_funcs(qpel_v, 8, 16, 64, sse4)
     390           0 : mc_rep_funcs(qpel_v, 8, 16, 48, sse4)
     391           0 : mc_rep_funcs(qpel_v, 8, 16, 32, sse4)
     392           0 : mc_rep_funcs(qpel_v, 8,  8, 24, sse4)
     393           0 : mc_rep_funcs(qpel_v,10,  8, 64, sse4)
     394           0 : mc_rep_funcs(qpel_v,10,  8, 48, sse4)
     395           0 : mc_rep_funcs(qpel_v,10,  8, 32, sse4)
     396           0 : mc_rep_funcs(qpel_v,10,  8, 24, sse4)
     397           0 : mc_rep_funcs(qpel_v,10,  8, 16, sse4)
     398           0 : mc_rep_funcs(qpel_v,10,  4, 12, sse4)
     399           0 : mc_rep_funcs(qpel_v,12,  8, 64, sse4)
     400           0 : mc_rep_funcs(qpel_v,12,  8, 48, sse4)
     401           0 : mc_rep_funcs(qpel_v,12,  8, 32, sse4)
     402           0 : mc_rep_funcs(qpel_v,12,  8, 24, sse4)
     403           0 : mc_rep_funcs(qpel_v,12,  8, 16, sse4)
     404           0 : mc_rep_funcs(qpel_v,12,  4, 12, sse4)
     405           0 : mc_rep_funcs(qpel_hv, 8,  8, 64, sse4)
     406           0 : mc_rep_funcs(qpel_hv, 8,  8, 48, sse4)
     407           0 : mc_rep_funcs(qpel_hv, 8,  8, 32, sse4)
     408           0 : mc_rep_funcs(qpel_hv, 8,  8, 24, sse4)
     409           0 : mc_rep_funcs(qpel_hv, 8,  8, 16, sse4)
     410           0 : mc_rep_funcs2(qpel_hv,8,  8,  4, 12, sse4)
     411           0 : mc_rep_funcs(qpel_hv,10,  8, 64, sse4)
     412           0 : mc_rep_funcs(qpel_hv,10,  8, 48, sse4)
     413           0 : mc_rep_funcs(qpel_hv,10,  8, 32, sse4)
     414           0 : mc_rep_funcs(qpel_hv,10,  8, 24, sse4)
     415           0 : mc_rep_funcs(qpel_hv,10,  8, 16, sse4)
     416           0 : mc_rep_funcs(qpel_hv,10,  4, 12, sse4)
     417           0 : mc_rep_funcs(qpel_hv,12,  8, 64, sse4)
     418           0 : mc_rep_funcs(qpel_hv,12,  8, 48, sse4)
     419           0 : mc_rep_funcs(qpel_hv,12,  8, 32, sse4)
     420           0 : mc_rep_funcs(qpel_hv,12,  8, 24, sse4)
     421           0 : mc_rep_funcs(qpel_hv,12,  8, 16, sse4)
     422           0 : mc_rep_funcs(qpel_hv,12,  4, 12, sse4)
     423             : 
     424             : #define mc_rep_uni_w(bitd, step, W, opt) \
     425             : void ff_hevc_put_hevc_uni_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, int16_t *_src, \
     426             :                                                int height, int denom,  int _wx, int _ox)                                \
     427             : {                                                                                                                       \
     428             :     int i;                                                                                                              \
     429             :     int16_t *src;                                                                                                       \
     430             :     uint8_t *dst;                                                                                                       \
     431             :     for (i = 0; i < W; i += step) {                                                                                     \
     432             :         src= _src + i;                                                                                                  \
     433             :         dst= _dst + (i * ((bitd + 7) / 8));                                                                             \
     434             :         ff_hevc_put_hevc_uni_w##step##_##bitd##_##opt(dst, dststride, src,                                   \
     435             :                                                      height, denom, _wx, _ox);                                          \
     436             :     }                                                                                                                   \
     437             : }
     438             : 
     439           0 : mc_rep_uni_w(8, 6, 12, sse4)
     440           0 : mc_rep_uni_w(8, 8, 16, sse4)
     441           0 : mc_rep_uni_w(8, 8, 24, sse4)
     442           0 : mc_rep_uni_w(8, 8, 32, sse4)
     443           0 : mc_rep_uni_w(8, 8, 48, sse4)
     444           0 : mc_rep_uni_w(8, 8, 64, sse4)
     445             : 
     446           0 : mc_rep_uni_w(10, 6, 12, sse4)
     447           0 : mc_rep_uni_w(10, 8, 16, sse4)
     448           0 : mc_rep_uni_w(10, 8, 24, sse4)
     449           0 : mc_rep_uni_w(10, 8, 32, sse4)
     450           0 : mc_rep_uni_w(10, 8, 48, sse4)
     451           0 : mc_rep_uni_w(10, 8, 64, sse4)
     452             : 
     453           0 : mc_rep_uni_w(12, 6, 12, sse4)
     454           0 : mc_rep_uni_w(12, 8, 16, sse4)
     455           0 : mc_rep_uni_w(12, 8, 24, sse4)
     456           0 : mc_rep_uni_w(12, 8, 32, sse4)
     457           0 : mc_rep_uni_w(12, 8, 48, sse4)
     458           0 : mc_rep_uni_w(12, 8, 64, sse4)
     459             : 
     460             : #define mc_rep_bi_w(bitd, step, W, opt) \
     461             : void ff_hevc_put_hevc_bi_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, int16_t *_src, \
     462             :                                               int16_t *_src2, int height,                                               \
     463             :                                               int denom,  int _wx0,  int _wx1, int _ox0, int _ox1)                      \
     464             : {                                                                                                                       \
     465             :     int i;                                                                                                              \
     466             :     int16_t *src;                                                                                                       \
     467             :     int16_t *src2;                                                                                                      \
     468             :     uint8_t *dst;                                                                                                       \
     469             :     for (i = 0; i < W; i += step) {                                                                                     \
     470             :         src  = _src  + i;                                                                                               \
     471             :         src2 = _src2 + i;                                                                                               \
     472             :         dst  = _dst  + (i * ((bitd + 7) / 8));                                                                          \
     473             :         ff_hevc_put_hevc_bi_w##step##_##bitd##_##opt(dst, dststride, src, src2,                             \
     474             :                                                      height, denom, _wx0, _wx1, _ox0, _ox1);                             \
     475             :     }                                                                                                                   \
     476             : }
     477             : 
     478           0 : mc_rep_bi_w(8, 6, 12, sse4)
     479           0 : mc_rep_bi_w(8, 8, 16, sse4)
     480           0 : mc_rep_bi_w(8, 8, 24, sse4)
     481           0 : mc_rep_bi_w(8, 8, 32, sse4)
     482           0 : mc_rep_bi_w(8, 8, 48, sse4)
     483           0 : mc_rep_bi_w(8, 8, 64, sse4)
     484             : 
     485           0 : mc_rep_bi_w(10, 6, 12, sse4)
     486           0 : mc_rep_bi_w(10, 8, 16, sse4)
     487           0 : mc_rep_bi_w(10, 8, 24, sse4)
     488           0 : mc_rep_bi_w(10, 8, 32, sse4)
     489           0 : mc_rep_bi_w(10, 8, 48, sse4)
     490           0 : mc_rep_bi_w(10, 8, 64, sse4)
     491             : 
     492           0 : mc_rep_bi_w(12, 6, 12, sse4)
     493           0 : mc_rep_bi_w(12, 8, 16, sse4)
     494           0 : mc_rep_bi_w(12, 8, 24, sse4)
     495           0 : mc_rep_bi_w(12, 8, 32, sse4)
     496           0 : mc_rep_bi_w(12, 8, 48, sse4)
     497           0 : mc_rep_bi_w(12, 8, 64, sse4)
     498             : 
     499             : #define mc_uni_w_func(name, bitd, W, opt) \
     500             : void ff_hevc_put_hevc_uni_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride,         \
     501             :                                                       uint8_t *_src, ptrdiff_t _srcstride,          \
     502             :                                                       int height, int denom,                        \
     503             :                                                       int _wx, int _ox,                             \
     504             :                                                       intptr_t mx, intptr_t my, int width)          \
     505             : {                                                                                                   \
     506             :     LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]);                                            \
     507             :     ff_hevc_put_hevc_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width);     \
     508             :     ff_hevc_put_hevc_uni_w##W##_##bitd##_##opt(_dst, _dststride, temp, height, denom, _wx, _ox);\
     509             : }
     510             : 
     511             : #define mc_uni_w_funcs(name, bitd, opt)      \
     512             :         mc_uni_w_func(name, bitd, 4, opt)    \
     513             :         mc_uni_w_func(name, bitd, 8, opt)    \
     514             :         mc_uni_w_func(name, bitd, 12, opt)   \
     515             :         mc_uni_w_func(name, bitd, 16, opt)   \
     516             :         mc_uni_w_func(name, bitd, 24, opt)   \
     517             :         mc_uni_w_func(name, bitd, 32, opt)   \
     518             :         mc_uni_w_func(name, bitd, 48, opt)   \
     519             :         mc_uni_w_func(name, bitd, 64, opt)
     520             : 
     521           0 : mc_uni_w_funcs(pel_pixels, 8, sse4)
     522           0 : mc_uni_w_func(pel_pixels, 8, 6, sse4)
     523           0 : mc_uni_w_funcs(epel_h, 8, sse4)
     524           0 : mc_uni_w_func(epel_h, 8, 6, sse4)
     525           0 : mc_uni_w_funcs(epel_v, 8, sse4)
     526           0 : mc_uni_w_func(epel_v, 8, 6, sse4)
     527           0 : mc_uni_w_funcs(epel_hv, 8, sse4)
     528           0 : mc_uni_w_func(epel_hv, 8, 6, sse4)
     529           0 : mc_uni_w_funcs(qpel_h, 8, sse4)
     530           0 : mc_uni_w_funcs(qpel_v, 8, sse4)
     531           0 : mc_uni_w_funcs(qpel_hv, 8, sse4)
     532             : 
     533           0 : mc_uni_w_funcs(pel_pixels, 10, sse4)
     534           0 : mc_uni_w_func(pel_pixels, 10, 6, sse4)
     535           0 : mc_uni_w_funcs(epel_h, 10, sse4)
     536           0 : mc_uni_w_func(epel_h, 10, 6, sse4)
     537           0 : mc_uni_w_funcs(epel_v, 10, sse4)
     538           0 : mc_uni_w_func(epel_v, 10, 6, sse4)
     539           0 : mc_uni_w_funcs(epel_hv, 10, sse4)
     540           0 : mc_uni_w_func(epel_hv, 10, 6, sse4)
     541           0 : mc_uni_w_funcs(qpel_h, 10, sse4)
     542           0 : mc_uni_w_funcs(qpel_v, 10, sse4)
     543           0 : mc_uni_w_funcs(qpel_hv, 10, sse4)
     544             : 
     545           0 : mc_uni_w_funcs(pel_pixels, 12, sse4)
     546           0 : mc_uni_w_func(pel_pixels, 12, 6, sse4)
     547           0 : mc_uni_w_funcs(epel_h, 12, sse4)
     548           0 : mc_uni_w_func(epel_h, 12, 6, sse4)
     549           0 : mc_uni_w_funcs(epel_v, 12, sse4)
     550           0 : mc_uni_w_func(epel_v, 12, 6, sse4)
     551           0 : mc_uni_w_funcs(epel_hv, 12, sse4)
     552           0 : mc_uni_w_func(epel_hv, 12, 6, sse4)
     553           0 : mc_uni_w_funcs(qpel_h, 12, sse4)
     554           0 : mc_uni_w_funcs(qpel_v, 12, sse4)
     555           0 : mc_uni_w_funcs(qpel_hv, 12, sse4)
     556             : 
     557             : #define mc_bi_w_func(name, bitd, W, opt) \
     558             : void ff_hevc_put_hevc_bi_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride,           \
     559             :                                                      uint8_t *_src, ptrdiff_t _srcstride,            \
     560             :                                                      int16_t *_src2,                                 \
     561             :                                                      int height, int denom,                          \
     562             :                                                      int _wx0, int _wx1, int _ox0, int _ox1,         \
     563             :                                                      intptr_t mx, intptr_t my, int width)            \
     564             : {                                                                                                    \
     565             :     LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]);                                             \
     566             :     ff_hevc_put_hevc_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width);      \
     567             :     ff_hevc_put_hevc_bi_w##W##_##bitd##_##opt(_dst, _dststride, temp, _src2,                         \
     568             :                                               height, denom, _wx0, _wx1, _ox0, _ox1);                \
     569             : }
     570             : 
     571             : #define mc_bi_w_funcs(name, bitd, opt)      \
     572             :         mc_bi_w_func(name, bitd, 4, opt)    \
     573             :         mc_bi_w_func(name, bitd, 8, opt)    \
     574             :         mc_bi_w_func(name, bitd, 12, opt)   \
     575             :         mc_bi_w_func(name, bitd, 16, opt)   \
     576             :         mc_bi_w_func(name, bitd, 24, opt)   \
     577             :         mc_bi_w_func(name, bitd, 32, opt)   \
     578             :         mc_bi_w_func(name, bitd, 48, opt)   \
     579             :         mc_bi_w_func(name, bitd, 64, opt)
     580             : 
     581           0 : mc_bi_w_funcs(pel_pixels, 8, sse4)
     582           0 : mc_bi_w_func(pel_pixels, 8, 6, sse4)
     583           0 : mc_bi_w_funcs(epel_h, 8, sse4)
     584           0 : mc_bi_w_func(epel_h, 8, 6, sse4)
     585           0 : mc_bi_w_funcs(epel_v, 8, sse4)
     586           0 : mc_bi_w_func(epel_v, 8, 6, sse4)
     587           0 : mc_bi_w_funcs(epel_hv, 8, sse4)
     588           0 : mc_bi_w_func(epel_hv, 8, 6, sse4)
     589           0 : mc_bi_w_funcs(qpel_h, 8, sse4)
     590           0 : mc_bi_w_funcs(qpel_v, 8, sse4)
     591           0 : mc_bi_w_funcs(qpel_hv, 8, sse4)
     592             : 
     593           0 : mc_bi_w_funcs(pel_pixels, 10, sse4)
     594           0 : mc_bi_w_func(pel_pixels, 10, 6, sse4)
     595           0 : mc_bi_w_funcs(epel_h, 10, sse4)
     596           0 : mc_bi_w_func(epel_h, 10, 6, sse4)
     597           0 : mc_bi_w_funcs(epel_v, 10, sse4)
     598           0 : mc_bi_w_func(epel_v, 10, 6, sse4)
     599           0 : mc_bi_w_funcs(epel_hv, 10, sse4)
     600           0 : mc_bi_w_func(epel_hv, 10, 6, sse4)
     601           0 : mc_bi_w_funcs(qpel_h, 10, sse4)
     602           0 : mc_bi_w_funcs(qpel_v, 10, sse4)
     603           0 : mc_bi_w_funcs(qpel_hv, 10, sse4)
     604             : 
     605           0 : mc_bi_w_funcs(pel_pixels, 12, sse4)
     606           0 : mc_bi_w_func(pel_pixels, 12, 6, sse4)
     607           0 : mc_bi_w_funcs(epel_h, 12, sse4)
     608           0 : mc_bi_w_func(epel_h, 12, 6, sse4)
     609           0 : mc_bi_w_funcs(epel_v, 12, sse4)
     610           0 : mc_bi_w_func(epel_v, 12, 6, sse4)
     611           0 : mc_bi_w_funcs(epel_hv, 12, sse4)
     612           0 : mc_bi_w_func(epel_hv, 12, 6, sse4)
     613           0 : mc_bi_w_funcs(qpel_h, 12, sse4)
     614           0 : mc_bi_w_funcs(qpel_v, 12, sse4)
     615           0 : mc_bi_w_funcs(qpel_hv, 12, sse4)
     616             : #endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL
     617             : 
     618             : #define SAO_BAND_FILTER_FUNCS(bitd, opt)                                                                                   \
     619             : void ff_hevc_sao_band_filter_8_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,  \
     620             :                                             int16_t *sao_offset_val, int sao_left_class, int width, int height);           \
     621             : void ff_hevc_sao_band_filter_16_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
     622             :                                             int16_t *sao_offset_val, int sao_left_class, int width, int height);           \
     623             : void ff_hevc_sao_band_filter_32_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
     624             :                                             int16_t *sao_offset_val, int sao_left_class, int width, int height);           \
     625             : void ff_hevc_sao_band_filter_48_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
     626             :                                             int16_t *sao_offset_val, int sao_left_class, int width, int height);           \
     627             : void ff_hevc_sao_band_filter_64_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
     628             :                                              int16_t *sao_offset_val, int sao_left_class, int width, int height);
     629             : 
     630             : SAO_BAND_FILTER_FUNCS(8,  sse2)
     631             : SAO_BAND_FILTER_FUNCS(10, sse2)
     632             : SAO_BAND_FILTER_FUNCS(12, sse2)
     633             : SAO_BAND_FILTER_FUNCS(8,   avx)
     634             : SAO_BAND_FILTER_FUNCS(10,  avx)
     635             : SAO_BAND_FILTER_FUNCS(12,  avx)
     636             : SAO_BAND_FILTER_FUNCS(8,  avx2)
     637             : SAO_BAND_FILTER_FUNCS(10, avx2)
     638             : SAO_BAND_FILTER_FUNCS(12, avx2)
     639             : 
     640             : #define SAO_BAND_INIT(bitd, opt) do {                                       \
     641             :     c->sao_band_filter[0]      = ff_hevc_sao_band_filter_8_##bitd##_##opt;  \
     642             :     c->sao_band_filter[1]      = ff_hevc_sao_band_filter_16_##bitd##_##opt; \
     643             :     c->sao_band_filter[2]      = ff_hevc_sao_band_filter_32_##bitd##_##opt; \
     644             :     c->sao_band_filter[3]      = ff_hevc_sao_band_filter_48_##bitd##_##opt; \
     645             :     c->sao_band_filter[4]      = ff_hevc_sao_band_filter_64_##bitd##_##opt; \
     646             : } while (0)
     647             : 
     648             : #define SAO_EDGE_FILTER_FUNCS(bitd, opt)                                                                                    \
     649             : void ff_hevc_sao_edge_filter_8_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val,  \
     650             :                                               int eo, int width, int height);                                               \
     651             : void ff_hevc_sao_edge_filter_16_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \
     652             :                                                int eo, int width, int height);                                              \
     653             : void ff_hevc_sao_edge_filter_32_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \
     654             :                                                int eo, int width, int height);                                              \
     655             : void ff_hevc_sao_edge_filter_48_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \
     656             :                                                int eo, int width, int height);                                              \
     657             : void ff_hevc_sao_edge_filter_64_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \
     658             :                                                int eo, int width, int height);                                              \
     659             : 
     660             : SAO_EDGE_FILTER_FUNCS(8, ssse3)
     661             : SAO_EDGE_FILTER_FUNCS(8, avx2)
     662             : SAO_EDGE_FILTER_FUNCS(10, sse2)
     663             : SAO_EDGE_FILTER_FUNCS(10, avx2)
     664             : SAO_EDGE_FILTER_FUNCS(12, sse2)
     665             : SAO_EDGE_FILTER_FUNCS(12, avx2)
     666             : 
     667             : #define SAO_EDGE_INIT(bitd, opt) do {                                       \
     668             :     c->sao_edge_filter[0]      = ff_hevc_sao_edge_filter_8_##bitd##_##opt;  \
     669             :     c->sao_edge_filter[1]      = ff_hevc_sao_edge_filter_16_##bitd##_##opt; \
     670             :     c->sao_edge_filter[2]      = ff_hevc_sao_edge_filter_32_##bitd##_##opt; \
     671             :     c->sao_edge_filter[3]      = ff_hevc_sao_edge_filter_48_##bitd##_##opt; \
     672             :     c->sao_edge_filter[4]      = ff_hevc_sao_edge_filter_64_##bitd##_##opt; \
     673             : } while (0)
     674             : 
     675             : #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt )           \
     676             :         PEL_LINK(pointer, 1, my , mx , fname##4 ,  bitd, opt ); \
     677             :         PEL_LINK(pointer, 2, my , mx , fname##6 ,  bitd, opt ); \
     678             :         PEL_LINK(pointer, 3, my , mx , fname##8 ,  bitd, opt ); \
     679             :         PEL_LINK(pointer, 4, my , mx , fname##12,  bitd, opt ); \
     680             :         PEL_LINK(pointer, 5, my , mx , fname##16,  bitd, opt ); \
     681             :         PEL_LINK(pointer, 6, my , mx , fname##24,  bitd, opt ); \
     682             :         PEL_LINK(pointer, 7, my , mx , fname##32,  bitd, opt ); \
     683             :         PEL_LINK(pointer, 8, my , mx , fname##48,  bitd, opt ); \
     684             :         PEL_LINK(pointer, 9, my , mx , fname##64,  bitd, opt )
     685             : #define QPEL_LINKS(pointer, my, mx, fname, bitd, opt)           \
     686             :         PEL_LINK(pointer, 1, my , mx , fname##4 ,  bitd, opt ); \
     687             :         PEL_LINK(pointer, 3, my , mx , fname##8 ,  bitd, opt ); \
     688             :         PEL_LINK(pointer, 4, my , mx , fname##12,  bitd, opt ); \
     689             :         PEL_LINK(pointer, 5, my , mx , fname##16,  bitd, opt ); \
     690             :         PEL_LINK(pointer, 6, my , mx , fname##24,  bitd, opt ); \
     691             :         PEL_LINK(pointer, 7, my , mx , fname##32,  bitd, opt ); \
     692             :         PEL_LINK(pointer, 8, my , mx , fname##48,  bitd, opt ); \
     693             :         PEL_LINK(pointer, 9, my , mx , fname##64,  bitd, opt )
     694             : 
     695         329 : void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
     696             : {
     697         329 :     int cpu_flags = av_get_cpu_flags();
     698             : 
     699         329 :     if (bit_depth == 8) {
     700         292 :         if (EXTERNAL_MMXEXT(cpu_flags)) {
     701           0 :             c->idct_dc[0] = ff_hevc_idct4x4_dc_8_mmxext;
     702           0 :             c->idct_dc[1] = ff_hevc_idct8x8_dc_8_mmxext;
     703           0 :             c->transform_add[0]    =  ff_hevc_transform_add4_8_mmxext;
     704             :         }
     705         292 :         if (EXTERNAL_SSE2(cpu_flags)) {
     706           0 :             c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_sse2;
     707           0 :             c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_sse2;
     708             :             if (ARCH_X86_64) {
     709           0 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2;
     710           0 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2;
     711             : 
     712             :             }
     713           0 :             SAO_BAND_INIT(8, sse2);
     714             : 
     715           0 :             c->idct_dc[1] = ff_hevc_idct8x8_dc_8_sse2;
     716           0 :             c->idct_dc[2] = ff_hevc_idct16x16_dc_8_sse2;
     717           0 :             c->idct_dc[3] = ff_hevc_idct32x32_dc_8_sse2;
     718             : 
     719           0 :             c->transform_add[1]    = ff_hevc_transform_add8_8_sse2;
     720           0 :             c->transform_add[2]    = ff_hevc_transform_add16_8_sse2;
     721           0 :             c->transform_add[3]    = ff_hevc_transform_add32_8_sse2;
     722             :         }
     723         292 :         if (EXTERNAL_SSSE3(cpu_flags)) {
     724             :             if(ARCH_X86_64) {
     725           0 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
     726           0 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
     727             :             }
     728           0 :             SAO_EDGE_INIT(8, ssse3);
     729             :         }
     730         292 :         if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
     731             : 
     732           0 :             EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels,  8, sse4);
     733           0 :             EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h,      8, sse4);
     734           0 :             EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v,      8, sse4);
     735           0 :             EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv,     8, sse4);
     736             : 
     737           0 :             QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 8, sse4);
     738           0 :             QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h,     8, sse4);
     739           0 :             QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v,     8, sse4);
     740           0 :             QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv,    8, sse4);
     741             :         }
     742         292 :         if (EXTERNAL_AVX(cpu_flags)) {
     743           0 :             c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_avx;
     744           0 :             c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_avx;
     745             :             if (ARCH_X86_64) {
     746           0 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_avx;
     747           0 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_avx;
     748             :             }
     749           0 :             SAO_BAND_INIT(8, avx);
     750             : 
     751           0 :             c->transform_add[1]    = ff_hevc_transform_add8_8_avx;
     752           0 :             c->transform_add[2]    = ff_hevc_transform_add16_8_avx;
     753           0 :             c->transform_add[3]    = ff_hevc_transform_add32_8_avx;
     754             :         }
     755         292 :         if (EXTERNAL_AVX2(cpu_flags)) {
     756           0 :             c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_8_avx2;
     757           0 :             c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_8_avx2;
     758             :         }
     759         292 :         if (EXTERNAL_AVX2_FAST(cpu_flags)) {
     760           0 :             c->idct_dc[2] = ff_hevc_idct16x16_dc_8_avx2;
     761           0 :             c->idct_dc[3] = ff_hevc_idct32x32_dc_8_avx2;
     762             :             if (ARCH_X86_64) {
     763           0 :                 c->put_hevc_epel[7][0][0] = ff_hevc_put_hevc_pel_pixels32_8_avx2;
     764           0 :                 c->put_hevc_epel[8][0][0] = ff_hevc_put_hevc_pel_pixels48_8_avx2;
     765           0 :                 c->put_hevc_epel[9][0][0] = ff_hevc_put_hevc_pel_pixels64_8_avx2;
     766             : 
     767           0 :                 c->put_hevc_qpel[7][0][0] = ff_hevc_put_hevc_pel_pixels32_8_avx2;
     768           0 :                 c->put_hevc_qpel[8][0][0] = ff_hevc_put_hevc_pel_pixels48_8_avx2;
     769           0 :                 c->put_hevc_qpel[9][0][0] = ff_hevc_put_hevc_pel_pixels64_8_avx2;
     770             : 
     771           0 :                 c->put_hevc_epel_uni[7][0][0] = ff_hevc_put_hevc_uni_pel_pixels32_8_avx2;
     772           0 :                 c->put_hevc_epel_uni[8][0][0] = ff_hevc_put_hevc_uni_pel_pixels48_8_avx2;
     773           0 :                 c->put_hevc_epel_uni[9][0][0] = ff_hevc_put_hevc_uni_pel_pixels64_8_avx2;
     774             : 
     775           0 :                 c->put_hevc_qpel_uni[7][0][0] = ff_hevc_put_hevc_uni_pel_pixels32_8_avx2;
     776           0 :                 c->put_hevc_qpel_uni[8][0][0] = ff_hevc_put_hevc_uni_pel_pixels48_8_avx2;
     777           0 :                 c->put_hevc_qpel_uni[9][0][0] = ff_hevc_put_hevc_uni_pel_pixels64_8_avx2;
     778             : 
     779           0 :                 c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_hevc_bi_pel_pixels32_8_avx2;
     780           0 :                 c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_hevc_bi_pel_pixels48_8_avx2;
     781           0 :                 c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_hevc_bi_pel_pixels64_8_avx2;
     782             : 
     783           0 :                 c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_hevc_bi_pel_pixels32_8_avx2;
     784           0 :                 c->put_hevc_epel_bi[8][0][0] = ff_hevc_put_hevc_bi_pel_pixels48_8_avx2;
     785           0 :                 c->put_hevc_epel_bi[9][0][0] = ff_hevc_put_hevc_bi_pel_pixels64_8_avx2;
     786             : 
     787           0 :                 c->put_hevc_epel[7][0][1] = ff_hevc_put_hevc_epel_h32_8_avx2;
     788           0 :                 c->put_hevc_epel[8][0][1] = ff_hevc_put_hevc_epel_h48_8_avx2;
     789           0 :                 c->put_hevc_epel[9][0][1] = ff_hevc_put_hevc_epel_h64_8_avx2;
     790             : 
     791           0 :                 c->put_hevc_epel_uni[7][0][1] = ff_hevc_put_hevc_uni_epel_h32_8_avx2;
     792           0 :                 c->put_hevc_epel_uni[8][0][1] = ff_hevc_put_hevc_uni_epel_h48_8_avx2;
     793           0 :                 c->put_hevc_epel_uni[9][0][1] = ff_hevc_put_hevc_uni_epel_h64_8_avx2;
     794             : 
     795           0 :                 c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_hevc_bi_epel_h32_8_avx2;
     796           0 :                 c->put_hevc_epel_bi[8][0][1] = ff_hevc_put_hevc_bi_epel_h48_8_avx2;
     797           0 :                 c->put_hevc_epel_bi[9][0][1] = ff_hevc_put_hevc_bi_epel_h64_8_avx2;
     798             : 
     799           0 :                 c->put_hevc_epel[7][1][0] = ff_hevc_put_hevc_epel_v32_8_avx2;
     800           0 :                 c->put_hevc_epel[8][1][0] = ff_hevc_put_hevc_epel_v48_8_avx2;
     801           0 :                 c->put_hevc_epel[9][1][0] = ff_hevc_put_hevc_epel_v64_8_avx2;
     802             : 
     803           0 :                 c->put_hevc_epel_uni[7][1][0] = ff_hevc_put_hevc_uni_epel_v32_8_avx2;
     804           0 :                 c->put_hevc_epel_uni[8][1][0] = ff_hevc_put_hevc_uni_epel_v48_8_avx2;
     805           0 :                 c->put_hevc_epel_uni[9][1][0] = ff_hevc_put_hevc_uni_epel_v64_8_avx2;
     806             : 
     807           0 :                 c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_hevc_bi_epel_v32_8_avx2;
     808           0 :                 c->put_hevc_epel_bi[8][1][0] = ff_hevc_put_hevc_bi_epel_v48_8_avx2;
     809           0 :                 c->put_hevc_epel_bi[9][1][0] = ff_hevc_put_hevc_bi_epel_v64_8_avx2;
     810             : 
     811           0 :                 c->put_hevc_epel[7][1][1] = ff_hevc_put_hevc_epel_hv32_8_avx2;
     812           0 :                 c->put_hevc_epel[8][1][1] = ff_hevc_put_hevc_epel_hv48_8_avx2;
     813           0 :                 c->put_hevc_epel[9][1][1] = ff_hevc_put_hevc_epel_hv64_8_avx2;
     814             : 
     815           0 :                 c->put_hevc_epel_uni[7][1][1] = ff_hevc_put_hevc_uni_epel_hv32_8_avx2;
     816           0 :                 c->put_hevc_epel_uni[8][1][1] = ff_hevc_put_hevc_uni_epel_hv48_8_avx2;
     817           0 :                 c->put_hevc_epel_uni[9][1][1] = ff_hevc_put_hevc_uni_epel_hv64_8_avx2;
     818             : 
     819           0 :                 c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_hevc_bi_epel_hv32_8_avx2;
     820           0 :                 c->put_hevc_epel_bi[8][1][1] = ff_hevc_put_hevc_bi_epel_hv48_8_avx2;
     821           0 :                 c->put_hevc_epel_bi[9][1][1] = ff_hevc_put_hevc_bi_epel_hv64_8_avx2;
     822             : 
     823           0 :                 c->put_hevc_qpel[7][0][1] = ff_hevc_put_hevc_qpel_h32_8_avx2;
     824           0 :                 c->put_hevc_qpel[8][0][1] = ff_hevc_put_hevc_qpel_h48_8_avx2;
     825           0 :                 c->put_hevc_qpel[9][0][1] = ff_hevc_put_hevc_qpel_h64_8_avx2;
     826             : 
     827           0 :                 c->put_hevc_qpel[7][1][0] = ff_hevc_put_hevc_qpel_v32_8_avx2;
     828           0 :                 c->put_hevc_qpel[8][1][0] = ff_hevc_put_hevc_qpel_v48_8_avx2;
     829           0 :                 c->put_hevc_qpel[9][1][0] = ff_hevc_put_hevc_qpel_v64_8_avx2;
     830             : 
     831           0 :                 c->put_hevc_qpel_uni[7][0][1] = ff_hevc_put_hevc_uni_qpel_h32_8_avx2;
     832           0 :                 c->put_hevc_qpel_uni[8][0][1] = ff_hevc_put_hevc_uni_qpel_h48_8_avx2;
     833           0 :                 c->put_hevc_qpel_uni[9][0][1] = ff_hevc_put_hevc_uni_qpel_h64_8_avx2;
     834             : 
     835           0 :                 c->put_hevc_qpel_uni[7][1][0] = ff_hevc_put_hevc_uni_qpel_v32_8_avx2;
     836           0 :                 c->put_hevc_qpel_uni[8][1][0] = ff_hevc_put_hevc_uni_qpel_v48_8_avx2;
     837           0 :                 c->put_hevc_qpel_uni[9][1][0] = ff_hevc_put_hevc_uni_qpel_v64_8_avx2;
     838             : 
     839           0 :                 c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_hevc_bi_qpel_h32_8_avx2;
     840           0 :                 c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_hevc_bi_qpel_h48_8_avx2;
     841           0 :                 c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_hevc_bi_qpel_h64_8_avx2;
     842             : 
     843           0 :                 c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_hevc_bi_qpel_v32_8_avx2;
     844           0 :                 c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_hevc_bi_qpel_v48_8_avx2;
     845           0 :                 c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_hevc_bi_qpel_v64_8_avx2;
     846             :             }
     847           0 :             SAO_BAND_INIT(8, avx2);
     848             : 
     849           0 :             c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_8_avx2;
     850           0 :             c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_8_avx2;
     851           0 :             c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_8_avx2;
     852             : 
     853           0 :             c->transform_add[3]    = ff_hevc_transform_add32_8_avx2;
     854             :         }
     855          37 :     } else if (bit_depth == 10) {
     856          30 :         if (EXTERNAL_MMXEXT(cpu_flags)) {
     857           1 :             c->transform_add[0] = ff_hevc_transform_add4_10_mmxext;
     858           1 :             c->idct_dc[0] = ff_hevc_idct4x4_dc_10_mmxext;
     859           1 :             c->idct_dc[1] = ff_hevc_idct8x8_dc_10_mmxext;
     860             :         }
     861          30 :         if (EXTERNAL_SSE2(cpu_flags)) {
     862           1 :             c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_sse2;
     863           1 :             c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_sse2;
     864             :             if (ARCH_X86_64) {
     865           1 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2;
     866           1 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2;
     867             :             }
     868           1 :             SAO_BAND_INIT(10, sse2);
     869           1 :             SAO_EDGE_INIT(10, sse2);
     870             : 
     871           1 :             c->idct_dc[1] = ff_hevc_idct8x8_dc_10_sse2;
     872           1 :             c->idct_dc[2] = ff_hevc_idct16x16_dc_10_sse2;
     873           1 :             c->idct_dc[3] = ff_hevc_idct32x32_dc_10_sse2;
     874             : 
     875           1 :             c->transform_add[1]    = ff_hevc_transform_add8_10_sse2;
     876           1 :             c->transform_add[2]    = ff_hevc_transform_add16_10_sse2;
     877           1 :             c->transform_add[3]    = ff_hevc_transform_add32_10_sse2;
     878             :         }
     879          30 :         if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
     880           1 :             c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_ssse3;
     881           1 :             c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_ssse3;
     882             :         }
     883          30 :         if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
     884           1 :             EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 10, sse4);
     885           1 :             EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h,     10, sse4);
     886           1 :             EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v,     10, sse4);
     887           1 :             EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv,    10, sse4);
     888             : 
     889           1 :             QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 10, sse4);
     890           1 :             QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h,     10, sse4);
     891           1 :             QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v,     10, sse4);
     892           1 :             QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv,    10, sse4);
     893             :         }
     894          30 :         if (EXTERNAL_AVX(cpu_flags)) {
     895           1 :             c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_avx;
     896           1 :             c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_avx;
     897             :             if (ARCH_X86_64) {
     898           1 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_avx;
     899           1 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_avx;
     900             :             }
     901           1 :             SAO_BAND_INIT(10, avx);
     902             :         }
     903          30 :         if (EXTERNAL_AVX2(cpu_flags)) {
     904           0 :             c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_10_avx2;
     905             :         }
     906          30 :         if (EXTERNAL_AVX2_FAST(cpu_flags)) {
     907           0 :             c->idct_dc[2] = ff_hevc_idct16x16_dc_10_avx2;
     908           0 :             c->idct_dc[3] = ff_hevc_idct32x32_dc_10_avx2;
     909             :             if (ARCH_X86_64) {
     910           0 :                 c->put_hevc_epel[5][0][0] = ff_hevc_put_hevc_pel_pixels16_10_avx2;
     911           0 :                 c->put_hevc_epel[6][0][0] = ff_hevc_put_hevc_pel_pixels24_10_avx2;
     912           0 :                 c->put_hevc_epel[7][0][0] = ff_hevc_put_hevc_pel_pixels32_10_avx2;
     913           0 :                 c->put_hevc_epel[8][0][0] = ff_hevc_put_hevc_pel_pixels48_10_avx2;
     914           0 :                 c->put_hevc_epel[9][0][0] = ff_hevc_put_hevc_pel_pixels64_10_avx2;
     915             : 
     916           0 :                 c->put_hevc_qpel[5][0][0] = ff_hevc_put_hevc_pel_pixels16_10_avx2;
     917           0 :                 c->put_hevc_qpel[6][0][0] = ff_hevc_put_hevc_pel_pixels24_10_avx2;
     918           0 :                 c->put_hevc_qpel[7][0][0] = ff_hevc_put_hevc_pel_pixels32_10_avx2;
     919           0 :                 c->put_hevc_qpel[8][0][0] = ff_hevc_put_hevc_pel_pixels48_10_avx2;
     920           0 :                 c->put_hevc_qpel[9][0][0] = ff_hevc_put_hevc_pel_pixels64_10_avx2;
     921             : 
     922           0 :                 c->put_hevc_epel_uni[5][0][0] = ff_hevc_put_hevc_uni_pel_pixels32_8_avx2;
     923           0 :                 c->put_hevc_epel_uni[6][0][0] = ff_hevc_put_hevc_uni_pel_pixels48_8_avx2;
     924           0 :                 c->put_hevc_epel_uni[7][0][0] = ff_hevc_put_hevc_uni_pel_pixels64_8_avx2;
     925           0 :                 c->put_hevc_epel_uni[8][0][0] = ff_hevc_put_hevc_uni_pel_pixels96_8_avx2;
     926           0 :                 c->put_hevc_epel_uni[9][0][0] = ff_hevc_put_hevc_uni_pel_pixels128_8_avx2;
     927             : 
     928           0 :                 c->put_hevc_qpel_uni[5][0][0] = ff_hevc_put_hevc_uni_pel_pixels32_8_avx2;
     929           0 :                 c->put_hevc_qpel_uni[6][0][0] = ff_hevc_put_hevc_uni_pel_pixels48_8_avx2;
     930           0 :                 c->put_hevc_qpel_uni[7][0][0] = ff_hevc_put_hevc_uni_pel_pixels64_8_avx2;
     931           0 :                 c->put_hevc_qpel_uni[8][0][0] = ff_hevc_put_hevc_uni_pel_pixels96_8_avx2;
     932           0 :                 c->put_hevc_qpel_uni[9][0][0] = ff_hevc_put_hevc_uni_pel_pixels128_8_avx2;
     933             : 
     934           0 :                 c->put_hevc_epel_bi[5][0][0] = ff_hevc_put_hevc_bi_pel_pixels16_10_avx2;
     935           0 :                 c->put_hevc_epel_bi[6][0][0] = ff_hevc_put_hevc_bi_pel_pixels24_10_avx2;
     936           0 :                 c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_hevc_bi_pel_pixels32_10_avx2;
     937           0 :                 c->put_hevc_epel_bi[8][0][0] = ff_hevc_put_hevc_bi_pel_pixels48_10_avx2;
     938           0 :                 c->put_hevc_epel_bi[9][0][0] = ff_hevc_put_hevc_bi_pel_pixels64_10_avx2;
     939           0 :                 c->put_hevc_qpel_bi[5][0][0] = ff_hevc_put_hevc_bi_pel_pixels16_10_avx2;
     940           0 :                 c->put_hevc_qpel_bi[6][0][0] = ff_hevc_put_hevc_bi_pel_pixels24_10_avx2;
     941           0 :                 c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_hevc_bi_pel_pixels32_10_avx2;
     942           0 :                 c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_hevc_bi_pel_pixels48_10_avx2;
     943           0 :                 c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_hevc_bi_pel_pixels64_10_avx2;
     944             : 
     945           0 :                 c->put_hevc_epel[5][0][1] = ff_hevc_put_hevc_epel_h16_10_avx2;
     946           0 :                 c->put_hevc_epel[6][0][1] = ff_hevc_put_hevc_epel_h24_10_avx2;
     947           0 :                 c->put_hevc_epel[7][0][1] = ff_hevc_put_hevc_epel_h32_10_avx2;
     948           0 :                 c->put_hevc_epel[8][0][1] = ff_hevc_put_hevc_epel_h48_10_avx2;
     949           0 :                 c->put_hevc_epel[9][0][1] = ff_hevc_put_hevc_epel_h64_10_avx2;
     950             : 
     951           0 :                 c->put_hevc_epel_uni[5][0][1] = ff_hevc_put_hevc_uni_epel_h16_10_avx2;
     952           0 :                 c->put_hevc_epel_uni[6][0][1] = ff_hevc_put_hevc_uni_epel_h24_10_avx2;
     953           0 :                 c->put_hevc_epel_uni[7][0][1] = ff_hevc_put_hevc_uni_epel_h32_10_avx2;
     954           0 :                 c->put_hevc_epel_uni[8][0][1] = ff_hevc_put_hevc_uni_epel_h48_10_avx2;
     955           0 :                 c->put_hevc_epel_uni[9][0][1] = ff_hevc_put_hevc_uni_epel_h64_10_avx2;
     956             : 
     957           0 :                 c->put_hevc_epel_bi[5][0][1] = ff_hevc_put_hevc_bi_epel_h16_10_avx2;
     958           0 :                 c->put_hevc_epel_bi[6][0][1] = ff_hevc_put_hevc_bi_epel_h24_10_avx2;
     959           0 :                 c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_hevc_bi_epel_h32_10_avx2;
     960           0 :                 c->put_hevc_epel_bi[8][0][1] = ff_hevc_put_hevc_bi_epel_h48_10_avx2;
     961           0 :                 c->put_hevc_epel_bi[9][0][1] = ff_hevc_put_hevc_bi_epel_h64_10_avx2;
     962             : 
     963           0 :                 c->put_hevc_epel[5][1][0] = ff_hevc_put_hevc_epel_v16_10_avx2;
     964           0 :                 c->put_hevc_epel[6][1][0] = ff_hevc_put_hevc_epel_v24_10_avx2;
     965           0 :                 c->put_hevc_epel[7][1][0] = ff_hevc_put_hevc_epel_v32_10_avx2;
     966           0 :                 c->put_hevc_epel[8][1][0] = ff_hevc_put_hevc_epel_v48_10_avx2;
     967           0 :                 c->put_hevc_epel[9][1][0] = ff_hevc_put_hevc_epel_v64_10_avx2;
     968             : 
     969           0 :                 c->put_hevc_epel_uni[5][1][0] = ff_hevc_put_hevc_uni_epel_v16_10_avx2;
     970           0 :                 c->put_hevc_epel_uni[6][1][0] = ff_hevc_put_hevc_uni_epel_v24_10_avx2;
     971           0 :                 c->put_hevc_epel_uni[7][1][0] = ff_hevc_put_hevc_uni_epel_v32_10_avx2;
     972           0 :                 c->put_hevc_epel_uni[8][1][0] = ff_hevc_put_hevc_uni_epel_v48_10_avx2;
     973           0 :                 c->put_hevc_epel_uni[9][1][0] = ff_hevc_put_hevc_uni_epel_v64_10_avx2;
     974             : 
     975           0 :                 c->put_hevc_epel_bi[5][1][0] = ff_hevc_put_hevc_bi_epel_v16_10_avx2;
     976           0 :                 c->put_hevc_epel_bi[6][1][0] = ff_hevc_put_hevc_bi_epel_v24_10_avx2;
     977           0 :                 c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_hevc_bi_epel_v32_10_avx2;
     978           0 :                 c->put_hevc_epel_bi[8][1][0] = ff_hevc_put_hevc_bi_epel_v48_10_avx2;
     979           0 :                 c->put_hevc_epel_bi[9][1][0] = ff_hevc_put_hevc_bi_epel_v64_10_avx2;
     980             : 
     981           0 :                 c->put_hevc_epel[5][1][1] = ff_hevc_put_hevc_epel_hv16_10_avx2;
     982           0 :                 c->put_hevc_epel[6][1][1] = ff_hevc_put_hevc_epel_hv24_10_avx2;
     983           0 :                 c->put_hevc_epel[7][1][1] = ff_hevc_put_hevc_epel_hv32_10_avx2;
     984           0 :                 c->put_hevc_epel[8][1][1] = ff_hevc_put_hevc_epel_hv48_10_avx2;
     985           0 :                 c->put_hevc_epel[9][1][1] = ff_hevc_put_hevc_epel_hv64_10_avx2;
     986             : 
     987           0 :                 c->put_hevc_epel_uni[5][1][1] = ff_hevc_put_hevc_uni_epel_hv16_10_avx2;
     988           0 :                 c->put_hevc_epel_uni[6][1][1] = ff_hevc_put_hevc_uni_epel_hv24_10_avx2;
     989           0 :                 c->put_hevc_epel_uni[7][1][1] = ff_hevc_put_hevc_uni_epel_hv32_10_avx2;
     990           0 :                 c->put_hevc_epel_uni[8][1][1] = ff_hevc_put_hevc_uni_epel_hv48_10_avx2;
     991           0 :                 c->put_hevc_epel_uni[9][1][1] = ff_hevc_put_hevc_uni_epel_hv64_10_avx2;
     992             : 
     993           0 :                 c->put_hevc_epel_bi[5][1][1] = ff_hevc_put_hevc_bi_epel_hv16_10_avx2;
     994           0 :                 c->put_hevc_epel_bi[6][1][1] = ff_hevc_put_hevc_bi_epel_hv24_10_avx2;
     995           0 :                 c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_hevc_bi_epel_hv32_10_avx2;
     996           0 :                 c->put_hevc_epel_bi[8][1][1] = ff_hevc_put_hevc_bi_epel_hv48_10_avx2;
     997           0 :                 c->put_hevc_epel_bi[9][1][1] = ff_hevc_put_hevc_bi_epel_hv64_10_avx2;
     998             : 
     999           0 :                 c->put_hevc_qpel[5][0][1] = ff_hevc_put_hevc_qpel_h16_10_avx2;
    1000           0 :                 c->put_hevc_qpel[6][0][1] = ff_hevc_put_hevc_qpel_h24_10_avx2;
    1001           0 :                 c->put_hevc_qpel[7][0][1] = ff_hevc_put_hevc_qpel_h32_10_avx2;
    1002           0 :                 c->put_hevc_qpel[8][0][1] = ff_hevc_put_hevc_qpel_h48_10_avx2;
    1003           0 :                 c->put_hevc_qpel[9][0][1] = ff_hevc_put_hevc_qpel_h64_10_avx2;
    1004             : 
    1005           0 :                 c->put_hevc_qpel_uni[5][0][1] = ff_hevc_put_hevc_uni_qpel_h16_10_avx2;
    1006           0 :                 c->put_hevc_qpel_uni[6][0][1] = ff_hevc_put_hevc_uni_qpel_h24_10_avx2;
    1007           0 :                 c->put_hevc_qpel_uni[7][0][1] = ff_hevc_put_hevc_uni_qpel_h32_10_avx2;
    1008           0 :                 c->put_hevc_qpel_uni[8][0][1] = ff_hevc_put_hevc_uni_qpel_h48_10_avx2;
    1009           0 :                 c->put_hevc_qpel_uni[9][0][1] = ff_hevc_put_hevc_uni_qpel_h64_10_avx2;
    1010             : 
    1011           0 :                 c->put_hevc_qpel_bi[5][0][1] = ff_hevc_put_hevc_bi_qpel_h16_10_avx2;
    1012           0 :                 c->put_hevc_qpel_bi[6][0][1] = ff_hevc_put_hevc_bi_qpel_h24_10_avx2;
    1013           0 :                 c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_hevc_bi_qpel_h32_10_avx2;
    1014           0 :                 c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_hevc_bi_qpel_h48_10_avx2;
    1015           0 :                 c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_hevc_bi_qpel_h64_10_avx2;
    1016             : 
    1017           0 :                 c->put_hevc_qpel[5][1][0] = ff_hevc_put_hevc_qpel_v16_10_avx2;
    1018           0 :                 c->put_hevc_qpel[6][1][0] = ff_hevc_put_hevc_qpel_v24_10_avx2;
    1019           0 :                 c->put_hevc_qpel[7][1][0] = ff_hevc_put_hevc_qpel_v32_10_avx2;
    1020           0 :                 c->put_hevc_qpel[8][1][0] = ff_hevc_put_hevc_qpel_v48_10_avx2;
    1021           0 :                 c->put_hevc_qpel[9][1][0] = ff_hevc_put_hevc_qpel_v64_10_avx2;
    1022             : 
    1023           0 :                 c->put_hevc_qpel_uni[5][1][0] = ff_hevc_put_hevc_uni_qpel_v16_10_avx2;
    1024           0 :                 c->put_hevc_qpel_uni[6][1][0] = ff_hevc_put_hevc_uni_qpel_v24_10_avx2;
    1025           0 :                 c->put_hevc_qpel_uni[7][1][0] = ff_hevc_put_hevc_uni_qpel_v32_10_avx2;
    1026           0 :                 c->put_hevc_qpel_uni[8][1][0] = ff_hevc_put_hevc_uni_qpel_v48_10_avx2;
    1027           0 :                 c->put_hevc_qpel_uni[9][1][0] = ff_hevc_put_hevc_uni_qpel_v64_10_avx2;
    1028             : 
    1029           0 :                 c->put_hevc_qpel_bi[5][1][0] = ff_hevc_put_hevc_bi_qpel_v16_10_avx2;
    1030           0 :                 c->put_hevc_qpel_bi[6][1][0] = ff_hevc_put_hevc_bi_qpel_v24_10_avx2;
    1031           0 :                 c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_hevc_bi_qpel_v32_10_avx2;
    1032           0 :                 c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_hevc_bi_qpel_v48_10_avx2;
    1033           0 :                 c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_hevc_bi_qpel_v64_10_avx2;
    1034             : 
    1035           0 :                 c->put_hevc_qpel[5][1][1] = ff_hevc_put_hevc_qpel_hv16_10_avx2;
    1036           0 :                 c->put_hevc_qpel[6][1][1] = ff_hevc_put_hevc_qpel_hv24_10_avx2;
    1037           0 :                 c->put_hevc_qpel[7][1][1] = ff_hevc_put_hevc_qpel_hv32_10_avx2;
    1038           0 :                 c->put_hevc_qpel[8][1][1] = ff_hevc_put_hevc_qpel_hv48_10_avx2;
    1039           0 :                 c->put_hevc_qpel[9][1][1] = ff_hevc_put_hevc_qpel_hv64_10_avx2;
    1040             : 
    1041           0 :                 c->put_hevc_qpel_uni[5][1][1] = ff_hevc_put_hevc_uni_qpel_hv16_10_avx2;
    1042           0 :                 c->put_hevc_qpel_uni[6][1][1] = ff_hevc_put_hevc_uni_qpel_hv24_10_avx2;
    1043           0 :                 c->put_hevc_qpel_uni[7][1][1] = ff_hevc_put_hevc_uni_qpel_hv32_10_avx2;
    1044           0 :                 c->put_hevc_qpel_uni[8][1][1] = ff_hevc_put_hevc_uni_qpel_hv48_10_avx2;
    1045           0 :                 c->put_hevc_qpel_uni[9][1][1] = ff_hevc_put_hevc_uni_qpel_hv64_10_avx2;
    1046             : 
    1047           0 :                 c->put_hevc_qpel_bi[5][1][1] = ff_hevc_put_hevc_bi_qpel_hv16_10_avx2;
    1048           0 :                 c->put_hevc_qpel_bi[6][1][1] = ff_hevc_put_hevc_bi_qpel_hv24_10_avx2;
    1049           0 :                 c->put_hevc_qpel_bi[7][1][1] = ff_hevc_put_hevc_bi_qpel_hv32_10_avx2;
    1050           0 :                 c->put_hevc_qpel_bi[8][1][1] = ff_hevc_put_hevc_bi_qpel_hv48_10_avx2;
    1051           0 :                 c->put_hevc_qpel_bi[9][1][1] = ff_hevc_put_hevc_bi_qpel_hv64_10_avx2;
    1052             :             }
    1053           0 :             SAO_BAND_INIT(10, avx2);
    1054           0 :             SAO_EDGE_INIT(10, avx2);
    1055             : 
    1056           0 :             c->transform_add[2] = ff_hevc_transform_add16_10_avx2;
    1057           0 :             c->transform_add[3] = ff_hevc_transform_add32_10_avx2;
    1058             : 
    1059             :         }
    1060           7 :     } else if (bit_depth == 12) {
    1061           7 :         if (EXTERNAL_MMXEXT(cpu_flags)) {
    1062           0 :             c->idct_dc[0] = ff_hevc_idct4x4_dc_12_mmxext;
    1063           0 :             c->idct_dc[1] = ff_hevc_idct8x8_dc_12_mmxext;
    1064             :         }
    1065           7 :         if (EXTERNAL_SSE2(cpu_flags)) {
    1066           0 :             c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_sse2;
    1067           0 :             c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_sse2;
    1068             :             if (ARCH_X86_64) {
    1069           0 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2;
    1070           0 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2;
    1071             :             }
    1072           0 :             SAO_BAND_INIT(12, sse2);
    1073           0 :             SAO_EDGE_INIT(12, sse2);
    1074             : 
    1075           0 :             c->idct_dc[1] = ff_hevc_idct8x8_dc_12_sse2;
    1076           0 :             c->idct_dc[2] = ff_hevc_idct16x16_dc_12_sse2;
    1077           0 :             c->idct_dc[3] = ff_hevc_idct32x32_dc_12_sse2;
    1078             :         }
    1079           7 :         if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
    1080           0 :             c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_ssse3;
    1081           0 :             c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_ssse3;
    1082             :         }
    1083           7 :         if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
    1084           0 :             EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 12, sse4);
    1085           0 :             EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h,     12, sse4);
    1086           0 :             EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v,     12, sse4);
    1087           0 :             EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv,    12, sse4);
    1088             : 
    1089           0 :             QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 12, sse4);
    1090           0 :             QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h,     12, sse4);
    1091           0 :             QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v,     12, sse4);
    1092           0 :             QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv,    12, sse4);
    1093             :         }
    1094           7 :         if (EXTERNAL_AVX(cpu_flags)) {
    1095           0 :             c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_avx;
    1096           0 :             c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_avx;
    1097             :             if (ARCH_X86_64) {
    1098           0 :                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_avx;
    1099           0 :                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_avx;
    1100             :             }
    1101           0 :             SAO_BAND_INIT(12, avx);
    1102             :         }
    1103           7 :         if (EXTERNAL_AVX2(cpu_flags)) {
    1104           0 :             c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_12_avx2;
    1105             :         }
    1106           7 :         if (EXTERNAL_AVX2_FAST(cpu_flags)) {
    1107           0 :             c->idct_dc[2] = ff_hevc_idct16x16_dc_12_avx2;
    1108           0 :             c->idct_dc[3] = ff_hevc_idct32x32_dc_12_avx2;
    1109             : 
    1110           0 :             SAO_BAND_INIT(12, avx2);
    1111           0 :             SAO_EDGE_INIT(12, avx2);
    1112             :         }
    1113             :     }
    1114         329 : }

Generated by: LCOV version 1.12