GCC Code Coverage Report
Directory: ../../../ffmpeg/ Exec Total Coverage
File: src/libavcodec/x86/h264_qpel.c Lines: 100 100 100.0 %
Date: 2021-04-18 10:33:33 Branches: 35 36 97.2 %

Line Branch Exec Source
1
/*
2
 * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
3
 * Copyright (c) 2011 Daniel Kang
4
 *
5
 * This file is part of FFmpeg.
6
 *
7
 * FFmpeg is free software; you can redistribute it and/or
8
 * modify it under the terms of the GNU Lesser General Public
9
 * License as published by the Free Software Foundation; either
10
 * version 2.1 of the License, or (at your option) any later version.
11
 *
12
 * FFmpeg is distributed in the hope that it will be useful,
13
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15
 * Lesser General Public License for more details.
16
 *
17
 * You should have received a copy of the GNU Lesser General Public
18
 * License along with FFmpeg; if not, write to the Free Software
19
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20
 */
21
22
#include "libavutil/attributes.h"
23
#include "libavutil/cpu.h"
24
#include "libavutil/mem_internal.h"
25
#include "libavutil/x86/asm.h"
26
#include "libavutil/x86/cpu.h"
27
#include "libavcodec/h264dec.h"
28
#include "libavcodec/h264qpel.h"
29
#include "libavcodec/pixels.h"
30
#include "fpel.h"
31
32
#if HAVE_X86ASM
33
void ff_put_pixels4_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
34
                              int dstStride, int src1Stride, int h);
35
void ff_avg_pixels4_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
36
                              int dstStride, int src1Stride, int h);
37
void ff_put_pixels8_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
38
                              int dstStride, int src1Stride, int h);
39
void ff_avg_pixels8_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
40
                              int dstStride, int src1Stride, int h);
41
void ff_put_pixels16_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
42
                               int dstStride, int src1Stride, int h);
43
void ff_avg_pixels16_l2_mmxext(uint8_t *dst, const uint8_t *src1, const uint8_t *src2,
44
                               int dstStride, int src1Stride, int h);
45
#define ff_put_pixels8_l2_sse2  ff_put_pixels8_l2_mmxext
46
#define ff_avg_pixels8_l2_sse2  ff_avg_pixels8_l2_mmxext
47
#define ff_put_pixels16_l2_sse2 ff_put_pixels16_l2_mmxext
48
#define ff_avg_pixels16_l2_sse2 ff_avg_pixels16_l2_mmxext
49
#define ff_put_pixels16_mmxext  ff_put_pixels16_mmx
50
#define ff_put_pixels8_mmxext   ff_put_pixels8_mmx
51
#define ff_put_pixels4_mmxext   ff_put_pixels4_mmx
52
53
#define DEF_QPEL(OPNAME)\
54
void ff_ ## OPNAME ## _h264_qpel4_h_lowpass_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
55
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
56
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_ssse3(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
57
void ff_ ## OPNAME ## _h264_qpel4_h_lowpass_l2_mmxext(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);\
58
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_l2_mmxext(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);\
59
void ff_ ## OPNAME ## _h264_qpel8_h_lowpass_l2_ssse3(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);\
60
void ff_ ## OPNAME ## _h264_qpel4_v_lowpass_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride);\
61
void ff_ ## OPNAME ## _h264_qpel8or16_v_lowpass_op_mmxext(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride, int h);\
62
void ff_ ## OPNAME ## _h264_qpel8or16_v_lowpass_sse2(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride, int h);\
63
void ff_ ## OPNAME ## _h264_qpel4_hv_lowpass_v_mmxext(const uint8_t *src, int16_t *tmp, int srcStride);\
64
void ff_ ## OPNAME ## _h264_qpel4_hv_lowpass_h_mmxext(int16_t *tmp, uint8_t *dst, int dstStride);\
65
void ff_ ## OPNAME ## _h264_qpel8or16_hv1_lowpass_op_mmxext(const uint8_t *src, int16_t *tmp, int srcStride, int size);\
66
void ff_ ## OPNAME ## _h264_qpel8or16_hv1_lowpass_op_sse2(const uint8_t *src, int16_t *tmp, int srcStride, int size);\
67
void ff_ ## OPNAME ## _h264_qpel8or16_hv2_lowpass_op_mmxext(uint8_t *dst, int16_t *tmp, int dstStride, int unused, int h);\
68
void ff_ ## OPNAME ## _h264_qpel8or16_hv2_lowpass_ssse3(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size);\
69
void ff_ ## OPNAME ## _pixels4_l2_shift5_mmxext(uint8_t *dst, const int16_t *src16, const uint8_t *src8, int dstStride, int src8Stride, int h);\
70
void ff_ ## OPNAME ## _pixels8_l2_shift5_mmxext(uint8_t *dst, const int16_t *src16, const uint8_t *src8, int dstStride, int src8Stride, int h);
71
72
DEF_QPEL(avg)
73
DEF_QPEL(put)
74
75
40
static av_always_inline void ff_put_h264_qpel8or16_hv1_lowpass_mmxext(int16_t *tmp, const uint8_t *src, int tmpStride, int srcStride, int size)
76
{
77
40
    int w = (size + 8) >> 2;
78
40
    src -= 2 * srcStride + 2;
79
240
    while (w--) {
80
200
        ff_put_h264_qpel8or16_hv1_lowpass_op_mmxext(src, tmp, srcStride, size);
81
200
        tmp += 4;
82
200
        src += 4;
83
    }
84
40
}
85
86
#define QPEL_H264(OPNAME, OP, MMX)\
87
static av_always_inline void ff_ ## OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
88
    int w=3;\
89
    src -= 2*srcStride+2;\
90
    while(w--){\
91
        ff_ ## OPNAME ## h264_qpel4_hv_lowpass_v_mmxext(src, tmp, srcStride);\
92
        tmp += 4;\
93
        src += 4;\
94
    }\
95
    tmp -= 3*4;\
96
    ff_ ## OPNAME ## h264_qpel4_hv_lowpass_h_mmxext(tmp, dst, dstStride);\
97
}\
98
\
99
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride, int h){\
100
    src -= 2*srcStride;\
101
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_op_mmxext(dst, src, dstStride, srcStride, h);\
102
    src += 4;\
103
    dst += 4;\
104
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_op_mmxext(dst, src, dstStride, srcStride, h);\
105
}\
106
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\
107
    int w = size>>4;\
108
    do{\
109
    ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_op_mmxext(dst, tmp, dstStride, 0, size);\
110
    tmp += 8;\
111
    dst += 8;\
112
    }while(w--);\
113
}\
114
\
115
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
116
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\
117
}\
118
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
119
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\
120
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\
121
}\
122
\
123
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
124
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
125
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
126
    src += 8*srcStride;\
127
    dst += 8*dstStride;\
128
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
129
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
130
}\
131
\
132
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride){\
133
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
134
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
135
    src += 8*dstStride;\
136
    dst += 8*dstStride;\
137
    src2 += 8*src2Stride;\
138
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
139
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
140
}\
141
\
142
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\
143
    ff_put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\
144
    ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\
145
}\
146
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
147
    ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 8);\
148
}\
149
\
150
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
151
    ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 16);\
152
}\
153
\
154
static av_always_inline void ff_ ## OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, const int16_t *src16, const uint8_t *src8, int dstStride, int src8Stride, int h)\
155
{\
156
    ff_ ## OPNAME ## pixels8_l2_shift5_ ## MMX(dst  , src16  , src8  , dstStride, src8Stride, h);\
157
    ff_ ## OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\
158
}\
159
160
161
#if ARCH_X86_64
162
#define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
163
164
void ff_avg_h264_qpel16_h_lowpass_l2_ssse3(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);
165
void ff_put_h264_qpel16_h_lowpass_l2_ssse3(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride);
166
167
#else // ARCH_X86_64
168
#define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
169
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, const uint8_t *src, const uint8_t *src2, int dstStride, int src2Stride){\
170
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
171
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
172
    src += 8*dstStride;\
173
    dst += 8*dstStride;\
174
    src2 += 8*src2Stride;\
175
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
176
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
177
}
178
#endif // ARCH_X86_64
179
180
#define QPEL_H264_H_XMM(OPNAME, OP, MMX)\
181
QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
182
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
183
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
184
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
185
    src += 8*srcStride;\
186
    dst += 8*dstStride;\
187
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
188
    ff_ ## OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
189
}\
190
191
#define QPEL_H264_V_XMM(OPNAME, OP, MMX)\
192
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
193
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\
194
}\
195
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, const uint8_t *src, int dstStride, int srcStride){\
196
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\
197
    ff_ ## OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\
198
}
199
200
74168
static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp,
201
                                                                 const uint8_t *src,
202
                                                                 int tmpStride,
203
                                                                 int srcStride,
204
                                                                 int size)
205
{
206
74168
    int w = (size+8)>>3;
207
74168
    src -= 2*srcStride+2;
208
240700
    while(w--){
209
166532
        ff_put_h264_qpel8or16_hv1_lowpass_op_sse2(src, tmp, srcStride, size);
210
166532
        tmp += 8;
211
166532
        src += 8;
212
    }
213
74168
}
214
215
#define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\
216
static av_always_inline void ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\
217
    put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\
218
    ff_ ## OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\
219
}\
220
static av_always_inline void ff_ ## OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
221
    ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\
222
}\
223
static av_always_inline void ff_ ## OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, const uint8_t *src, int dstStride, int tmpStride, int srcStride){\
224
    ff_ ## OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\
225
}\
226
227
#define ff_put_h264_qpel8_h_lowpass_l2_sse2  ff_put_h264_qpel8_h_lowpass_l2_mmxext
228
#define ff_avg_h264_qpel8_h_lowpass_l2_sse2  ff_avg_h264_qpel8_h_lowpass_l2_mmxext
229
#define ff_put_h264_qpel16_h_lowpass_l2_sse2 ff_put_h264_qpel16_h_lowpass_l2_mmxext
230
#define ff_avg_h264_qpel16_h_lowpass_l2_sse2 ff_avg_h264_qpel16_h_lowpass_l2_mmxext
231
232
#define ff_put_h264_qpel8_v_lowpass_ssse3  ff_put_h264_qpel8_v_lowpass_sse2
233
#define ff_avg_h264_qpel8_v_lowpass_ssse3  ff_avg_h264_qpel8_v_lowpass_sse2
234
#define ff_put_h264_qpel16_v_lowpass_ssse3 ff_put_h264_qpel16_v_lowpass_sse2
235
#define ff_avg_h264_qpel16_v_lowpass_ssse3 ff_avg_h264_qpel16_v_lowpass_sse2
236
237
#define ff_put_h264_qpel8or16_hv2_lowpass_sse2 ff_put_h264_qpel8or16_hv2_lowpass_mmxext
238
#define ff_avg_h264_qpel8or16_hv2_lowpass_sse2 ff_avg_h264_qpel8or16_hv2_lowpass_mmxext
239
240
#define H264_MC(OPNAME, SIZE, MMX, ALIGN) \
241
H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\
242
H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\
243
H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\
244
H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\
245
246
219947
static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, const uint8_t *src,
247
                                       ptrdiff_t stride)
248
{
249
219947
    ff_put_pixels16_sse2(dst, src, stride, 16);
250
219947
}
251
149898
static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, const uint8_t *src,
252
                                       ptrdiff_t stride)
253
{
254
149898
    ff_avg_pixels16_sse2(dst, src, stride, 16);
255
149898
}
256
#define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmxext
257
#define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmxext
258
259
#define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \
260
static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
261
{\
262
    ff_ ## OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\
263
}\
264
265
#define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \
266
static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
267
{\
268
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\
269
}\
270
\
271
static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
272
{\
273
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\
274
}\
275
\
276
static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
277
{\
278
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\
279
}\
280
281
#define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \
282
static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
283
{\
284
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
285
    ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
286
    ff_ ## OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\
287
}\
288
\
289
static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
290
{\
291
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\
292
}\
293
\
294
static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
295
{\
296
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
297
    ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
298
    ff_ ## OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\
299
}\
300
301
#define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \
302
static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
303
{\
304
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
305
    ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
306
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\
307
}\
308
\
309
static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
310
{\
311
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
312
    ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\
313
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\
314
}\
315
\
316
static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
317
{\
318
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
319
    ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
320
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\
321
}\
322
\
323
static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
324
{\
325
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*SIZE]);\
326
    ff_put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\
327
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\
328
}\
329
\
330
static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
331
{\
332
    LOCAL_ALIGNED(ALIGN, uint16_t, temp, [SIZE*(SIZE<8?12:24)]);\
333
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\
334
}\
335
\
336
static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
337
{\
338
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
339
    uint8_t * const halfHV= temp;\
340
    int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
341
    av_assert2(((int)temp & 7) == 0);\
342
    ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
343
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\
344
}\
345
\
346
static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
347
{\
348
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
349
    uint8_t * const halfHV= temp;\
350
    int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
351
    av_assert2(((int)temp & 7) == 0);\
352
    ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
353
    ff_ ## OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\
354
}\
355
\
356
static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
357
{\
358
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
359
    uint8_t * const halfHV= temp;\
360
    int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
361
    av_assert2(((int)temp & 7) == 0);\
362
    ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
363
    ff_ ## OPNAME ## pixels ## SIZE ## _l2_shift5_mmxext(dst, halfV+2, halfHV, stride, SIZE, SIZE);\
364
}\
365
\
366
static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride)\
367
{\
368
    LOCAL_ALIGNED(ALIGN, uint8_t, temp, [SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE]);\
369
    uint8_t * const halfHV= temp;\
370
    int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
371
    av_assert2(((int)temp & 7) == 0);\
372
    ff_put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
373
    ff_ ## OPNAME ## pixels ## SIZE ## _l2_shift5_mmxext(dst, halfV+3, halfHV, stride, SIZE, SIZE);\
374
}\
375
376
#define H264_MC_4816(MMX)\
377
H264_MC(put_, 4, MMX, 8)\
378
H264_MC(put_, 8, MMX, 8)\
379
H264_MC(put_, 16,MMX, 8)\
380
H264_MC(avg_, 4, MMX, 8)\
381
H264_MC(avg_, 8, MMX, 8)\
382
H264_MC(avg_, 16,MMX, 8)\
383
384
#define H264_MC_816(QPEL, XMM)\
385
QPEL(put_, 8, XMM, 16)\
386
QPEL(put_, 16,XMM, 16)\
387
QPEL(avg_, 8, XMM, 16)\
388
QPEL(avg_, 16,XMM, 16)\
389
390
16402
QPEL_H264(put_,        PUT_OP, mmxext)
391
207
QPEL_H264(avg_, AVG_MMXEXT_OP, mmxext)
392
156140
QPEL_H264_V_XMM(put_,       PUT_OP, sse2)
393
210
QPEL_H264_V_XMM(avg_,AVG_MMXEXT_OP, sse2)
394
72
QPEL_H264_HV_XMM(put_,       PUT_OP, sse2)
395
8
QPEL_H264_HV_XMM(avg_,AVG_MMXEXT_OP, sse2)
396
5871
QPEL_H264_H_XMM(put_,       PUT_OP, ssse3)
397
624
QPEL_H264_H_XMM(avg_,AVG_MMXEXT_OP, ssse3)
398
296152
QPEL_H264_HV_XMM(put_,       PUT_OP, ssse3)
399
180
QPEL_H264_HV_XMM(avg_,AVG_MMXEXT_OP, ssse3)
400
401
55874
H264_MC_4816(mmxext)
402
54238
H264_MC_816(H264_MC_V, sse2)
403
72
H264_MC_816(H264_MC_HV, sse2)
404
61168
H264_MC_816(H264_MC_H, ssse3)
405
176208
H264_MC_816(H264_MC_HV, ssse3)
406
407
408
//10bit
409
#define LUMA_MC_OP(OP, NUM, DEPTH, TYPE, OPT) \
410
void ff_ ## OP ## _h264_qpel ## NUM ## _ ## TYPE ## _ ## DEPTH ## _ ## OPT \
411
    (uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
412
413
#define LUMA_MC_ALL(DEPTH, TYPE, OPT) \
414
    LUMA_MC_OP(put,  4, DEPTH, TYPE, OPT) \
415
    LUMA_MC_OP(avg,  4, DEPTH, TYPE, OPT) \
416
    LUMA_MC_OP(put,  8, DEPTH, TYPE, OPT) \
417
    LUMA_MC_OP(avg,  8, DEPTH, TYPE, OPT) \
418
    LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \
419
    LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)
420
421
#define LUMA_MC_816(DEPTH, TYPE, OPT) \
422
    LUMA_MC_OP(put,  8, DEPTH, TYPE, OPT) \
423
    LUMA_MC_OP(avg,  8, DEPTH, TYPE, OPT) \
424
    LUMA_MC_OP(put, 16, DEPTH, TYPE, OPT) \
425
    LUMA_MC_OP(avg, 16, DEPTH, TYPE, OPT)
426
427
LUMA_MC_ALL(10, mc00, mmxext)
428
LUMA_MC_ALL(10, mc10, mmxext)
429
LUMA_MC_ALL(10, mc20, mmxext)
430
LUMA_MC_ALL(10, mc30, mmxext)
431
LUMA_MC_ALL(10, mc01, mmxext)
432
LUMA_MC_ALL(10, mc11, mmxext)
433
LUMA_MC_ALL(10, mc21, mmxext)
434
LUMA_MC_ALL(10, mc31, mmxext)
435
LUMA_MC_ALL(10, mc02, mmxext)
436
LUMA_MC_ALL(10, mc12, mmxext)
437
LUMA_MC_ALL(10, mc22, mmxext)
438
LUMA_MC_ALL(10, mc32, mmxext)
439
LUMA_MC_ALL(10, mc03, mmxext)
440
LUMA_MC_ALL(10, mc13, mmxext)
441
LUMA_MC_ALL(10, mc23, mmxext)
442
LUMA_MC_ALL(10, mc33, mmxext)
443
444
LUMA_MC_816(10, mc00, sse2)
445
LUMA_MC_816(10, mc10, sse2)
446
LUMA_MC_816(10, mc10, sse2_cache64)
447
LUMA_MC_816(10, mc10, ssse3_cache64)
448
LUMA_MC_816(10, mc20, sse2)
449
LUMA_MC_816(10, mc20, sse2_cache64)
450
LUMA_MC_816(10, mc20, ssse3_cache64)
451
LUMA_MC_816(10, mc30, sse2)
452
LUMA_MC_816(10, mc30, sse2_cache64)
453
LUMA_MC_816(10, mc30, ssse3_cache64)
454
LUMA_MC_816(10, mc01, sse2)
455
LUMA_MC_816(10, mc11, sse2)
456
LUMA_MC_816(10, mc21, sse2)
457
LUMA_MC_816(10, mc31, sse2)
458
LUMA_MC_816(10, mc02, sse2)
459
LUMA_MC_816(10, mc12, sse2)
460
LUMA_MC_816(10, mc22, sse2)
461
LUMA_MC_816(10, mc32, sse2)
462
LUMA_MC_816(10, mc03, sse2)
463
LUMA_MC_816(10, mc13, sse2)
464
LUMA_MC_816(10, mc23, sse2)
465
LUMA_MC_816(10, mc33, sse2)
466
467
#define QPEL16_OPMC(OP, MC, MMX)\
468
void ff_ ## OP ## _h264_qpel16_ ## MC ## _10_ ## MMX(uint8_t *dst, const uint8_t *src, ptrdiff_t stride){\
469
    ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst   , src   , stride);\
470
    ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\
471
    src += 8*stride;\
472
    dst += 8*stride;\
473
    ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst   , src   , stride);\
474
    ff_ ## OP ## _h264_qpel8_ ## MC ## _10_ ## MMX(dst+16, src+16, stride);\
475
}
476
477
#define QPEL16_OP(MC, MMX)\
478
QPEL16_OPMC(put, MC, MMX)\
479
QPEL16_OPMC(avg, MC, MMX)
480
481
#define QPEL16(MMX)\
482
QPEL16_OP(mc00, MMX)\
483
QPEL16_OP(mc01, MMX)\
484
QPEL16_OP(mc02, MMX)\
485
QPEL16_OP(mc03, MMX)\
486
QPEL16_OP(mc10, MMX)\
487
QPEL16_OP(mc11, MMX)\
488
QPEL16_OP(mc12, MMX)\
489
QPEL16_OP(mc13, MMX)\
490
QPEL16_OP(mc20, MMX)\
491
QPEL16_OP(mc21, MMX)\
492
QPEL16_OP(mc22, MMX)\
493
QPEL16_OP(mc23, MMX)\
494
QPEL16_OP(mc30, MMX)\
495
QPEL16_OP(mc31, MMX)\
496
QPEL16_OP(mc32, MMX)\
497
QPEL16_OP(mc33, MMX)
498
499
#if ARCH_X86_32 // ARCH_X86_64 implies SSE2+
500
QPEL16(mmxext)
501
#endif
502
503
#endif /* HAVE_X86ASM */
504
505
#define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
506
    do {                                                                     \
507
    c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
508
    c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
509
    c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
510
    c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
511
    c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
512
    c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
513
    c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
514
    c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
515
    c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
516
    c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
517
    c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
518
    c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
519
    c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
520
    c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
521
    c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
522
    c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
523
    } while (0)
524
525
#define H264_QPEL_FUNCS(x, y, CPU)                                                            \
526
    do {                                                                                      \
527
        c->put_h264_qpel_pixels_tab[0][x + y * 4] = put_h264_qpel16_mc ## x ## y ## _ ## CPU; \
528
        c->put_h264_qpel_pixels_tab[1][x + y * 4] = put_h264_qpel8_mc  ## x ## y ## _ ## CPU; \
529
        c->avg_h264_qpel_pixels_tab[0][x + y * 4] = avg_h264_qpel16_mc ## x ## y ## _ ## CPU; \
530
        c->avg_h264_qpel_pixels_tab[1][x + y * 4] = avg_h264_qpel8_mc  ## x ## y ## _ ## CPU; \
531
    } while (0)
532
533
#define H264_QPEL_FUNCS_10(x, y, CPU)                                                               \
534
    do {                                                                                            \
535
        c->put_h264_qpel_pixels_tab[0][x + y * 4] = ff_put_h264_qpel16_mc ## x ## y ## _10_ ## CPU; \
536
        c->put_h264_qpel_pixels_tab[1][x + y * 4] = ff_put_h264_qpel8_mc  ## x ## y ## _10_ ## CPU; \
537
        c->avg_h264_qpel_pixels_tab[0][x + y * 4] = ff_avg_h264_qpel16_mc ## x ## y ## _10_ ## CPU; \
538
        c->avg_h264_qpel_pixels_tab[1][x + y * 4] = ff_avg_h264_qpel8_mc  ## x ## y ## _10_ ## CPU; \
539
    } while (0)
540
541
664
av_cold void ff_h264qpel_init_x86(H264QpelContext *c, int bit_depth)
542
{
543
#if HAVE_X86ASM
544
664
    int high_bit_depth = bit_depth > 8;
545
664
    int cpu_flags = av_get_cpu_flags();
546
547
664
    if (EXTERNAL_MMXEXT(cpu_flags)) {
548
104
        if (!high_bit_depth) {
549
60
            SET_QPEL_FUNCS(put_h264_qpel, 0, 16, mmxext, );
550
60
            SET_QPEL_FUNCS(put_h264_qpel, 1,  8, mmxext, );
551
60
            SET_QPEL_FUNCS(put_h264_qpel, 2,  4, mmxext, );
552
60
            SET_QPEL_FUNCS(avg_h264_qpel, 0, 16, mmxext, );
553
60
            SET_QPEL_FUNCS(avg_h264_qpel, 1,  8, mmxext, );
554
60
            SET_QPEL_FUNCS(avg_h264_qpel, 2,  4, mmxext, );
555
44
        } else if (bit_depth == 10) {
556
#if ARCH_X86_32
557
            SET_QPEL_FUNCS(avg_h264_qpel, 0, 16, 10_mmxext, ff_);
558
            SET_QPEL_FUNCS(put_h264_qpel, 0, 16, 10_mmxext, ff_);
559
            SET_QPEL_FUNCS(put_h264_qpel, 1,  8, 10_mmxext, ff_);
560
            SET_QPEL_FUNCS(avg_h264_qpel, 1,  8, 10_mmxext, ff_);
561
#endif
562
22
            SET_QPEL_FUNCS(put_h264_qpel, 2, 4,  10_mmxext, ff_);
563
22
            SET_QPEL_FUNCS(avg_h264_qpel, 2, 4,  10_mmxext, ff_);
564
        }
565
    }
566
567
664
    if (EXTERNAL_SSE2(cpu_flags)) {
568
92
        if (!high_bit_depth) {
569
56
            H264_QPEL_FUNCS(0, 1, sse2);
570
56
            H264_QPEL_FUNCS(0, 2, sse2);
571
56
            H264_QPEL_FUNCS(0, 3, sse2);
572
56
            H264_QPEL_FUNCS(1, 1, sse2);
573
56
            H264_QPEL_FUNCS(1, 2, sse2);
574
56
            H264_QPEL_FUNCS(1, 3, sse2);
575
56
            H264_QPEL_FUNCS(2, 1, sse2);
576
56
            H264_QPEL_FUNCS(2, 2, sse2);
577
56
            H264_QPEL_FUNCS(2, 3, sse2);
578
56
            H264_QPEL_FUNCS(3, 1, sse2);
579
56
            H264_QPEL_FUNCS(3, 2, sse2);
580
56
            H264_QPEL_FUNCS(3, 3, sse2);
581
        }
582
583
92
        if (bit_depth == 10) {
584
18
            SET_QPEL_FUNCS(put_h264_qpel, 0, 16, 10_sse2, ff_);
585
18
            SET_QPEL_FUNCS(put_h264_qpel, 1,  8, 10_sse2, ff_);
586
18
            SET_QPEL_FUNCS(avg_h264_qpel, 0, 16, 10_sse2, ff_);
587
18
            SET_QPEL_FUNCS(avg_h264_qpel, 1,  8, 10_sse2, ff_);
588
18
            H264_QPEL_FUNCS_10(1, 0, sse2_cache64);
589
18
            H264_QPEL_FUNCS_10(2, 0, sse2_cache64);
590
18
            H264_QPEL_FUNCS_10(3, 0, sse2_cache64);
591
        }
592
    }
593
594

664
    if (EXTERNAL_SSE2_FAST(cpu_flags)) {
595
92
        if (!high_bit_depth) {
596
56
            H264_QPEL_FUNCS(0, 0, sse2);
597
        }
598
    }
599
600
664
    if (EXTERNAL_SSSE3(cpu_flags)) {
601
80
        if (!high_bit_depth) {
602
52
            H264_QPEL_FUNCS(1, 0, ssse3);
603
52
            H264_QPEL_FUNCS(1, 1, ssse3);
604
52
            H264_QPEL_FUNCS(1, 2, ssse3);
605
52
            H264_QPEL_FUNCS(1, 3, ssse3);
606
52
            H264_QPEL_FUNCS(2, 0, ssse3);
607
52
            H264_QPEL_FUNCS(2, 1, ssse3);
608
52
            H264_QPEL_FUNCS(2, 2, ssse3);
609
52
            H264_QPEL_FUNCS(2, 3, ssse3);
610
52
            H264_QPEL_FUNCS(3, 0, ssse3);
611
52
            H264_QPEL_FUNCS(3, 1, ssse3);
612
52
            H264_QPEL_FUNCS(3, 2, ssse3);
613
52
            H264_QPEL_FUNCS(3, 3, ssse3);
614
        }
615
616
80
        if (bit_depth == 10) {
617
14
            H264_QPEL_FUNCS_10(1, 0, ssse3_cache64);
618
14
            H264_QPEL_FUNCS_10(2, 0, ssse3_cache64);
619
14
            H264_QPEL_FUNCS_10(3, 0, ssse3_cache64);
620
        }
621
    }
622
623
664
    if (EXTERNAL_AVX(cpu_flags)) {
624
        /* AVX implies 64 byte cache lines without the need to avoid unaligned
625
         * memory accesses that cross the boundary between two cache lines.
626
         * TODO: Port X264_CPU_CACHELINE_32/64 detection from x264 to avoid
627
         * having to treat SSE2 functions with such properties as AVX. */
628
56
        if (bit_depth == 10) {
629
6
            H264_QPEL_FUNCS_10(1, 0, sse2);
630
6
            H264_QPEL_FUNCS_10(2, 0, sse2);
631
6
            H264_QPEL_FUNCS_10(3, 0, sse2);
632
        }
633
    }
634
#endif
635
664
}