1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <emmintrin.h> // SSE2
13
14 #include "config/aom_dsp_rtcd.h"
15
16 #include "aom_dsp/txfm_common.h"
17 #include "aom_dsp/x86/fwd_txfm_sse2.h"
18 #include "aom_dsp/x86/txfm_common_sse2.h"
19 #include "aom_ports/mem.h"
20
21 // TODO(jingning) The high bit-depth functions need rework for performance.
22 // After we properly fix the high bit-depth function implementations, this
23 // file's dependency should be substantially simplified.
24 #if DCT_HIGH_BIT_DEPTH
25 #define ADD_EPI16 _mm_adds_epi16
26 #define SUB_EPI16 _mm_subs_epi16
27
28 #else
29 #define ADD_EPI16 _mm_add_epi16
30 #define SUB_EPI16 _mm_sub_epi16
31 #endif
32
33 #if defined(FDCT4x4_2D_HELPER)
FDCT4x4_2D_HELPER(const int16_t * input,int stride,__m128i * in0,__m128i * in1)34 static void FDCT4x4_2D_HELPER(const int16_t *input, int stride, __m128i *in0,
35 __m128i *in1) {
36 // Constants
37 // These are the coefficients used for the multiplies.
38 // In the comments, pN means cos(N pi /64) and mN is -cos(N pi /64),
39 // where cospi_N_64 = cos(N pi /64)
40 const __m128i k__cospi_A =
41 octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64,
42 cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64);
43 const __m128i k__cospi_B =
44 octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64,
45 cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64);
46 const __m128i k__cospi_C =
47 octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64,
48 cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64);
49 const __m128i k__cospi_D =
50 octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64,
51 cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64);
52 const __m128i k__cospi_E =
53 octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64,
54 cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64);
55 const __m128i k__cospi_F =
56 octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64,
57 cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64);
58 const __m128i k__cospi_G =
59 octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64,
60 -cospi_8_64, -cospi_24_64, -cospi_8_64, -cospi_24_64);
61 const __m128i k__cospi_H =
62 octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64,
63 -cospi_24_64, cospi_8_64, -cospi_24_64, cospi_8_64);
64
65 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
66 // This second rounding constant saves doing some extra adds at the end
67 const __m128i k__DCT_CONST_ROUNDING2 =
68 _mm_set1_epi32(DCT_CONST_ROUNDING + (DCT_CONST_ROUNDING << 1));
69 const int DCT_CONST_BITS2 = DCT_CONST_BITS + 2;
70 const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
71 const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
72
73 // Load inputs.
74 *in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride));
75 *in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride));
76 *in1 = _mm_unpacklo_epi64(
77 *in1, _mm_loadl_epi64((const __m128i *)(input + 2 * stride)));
78 *in0 = _mm_unpacklo_epi64(
79 *in0, _mm_loadl_epi64((const __m128i *)(input + 3 * stride)));
80 // in0 = [i0 i1 i2 i3 iC iD iE iF]
81 // in1 = [i4 i5 i6 i7 i8 i9 iA iB]
82 // multiply by 16 to give some extra precision
83 *in0 = _mm_slli_epi16(*in0, 4);
84 *in1 = _mm_slli_epi16(*in1, 4);
85 // if (i == 0 && input[0]) input[0] += 1;
86 // add 1 to the upper left pixel if it is non-zero, which helps reduce
87 // the round-trip error
88 {
89 // The mask will only contain whether the first value is zero, all
90 // other comparison will fail as something shifted by 4 (above << 4)
91 // can never be equal to one. To increment in the non-zero case, we
92 // add the mask and one for the first element:
93 // - if zero, mask = -1, v = v - 1 + 1 = v
94 // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1
95 __m128i mask = _mm_cmpeq_epi16(*in0, k__nonzero_bias_a);
96 *in0 = _mm_add_epi16(*in0, mask);
97 *in0 = _mm_add_epi16(*in0, k__nonzero_bias_b);
98 }
99 // There are 4 total stages, alternating between an add/subtract stage
100 // followed by an multiply-and-add stage.
101 {
102 // Stage 1: Add/subtract
103
104 // in0 = [i0 i1 i2 i3 iC iD iE iF]
105 // in1 = [i4 i5 i6 i7 i8 i9 iA iB]
106 const __m128i r0 = _mm_unpacklo_epi16(*in0, *in1);
107 const __m128i r1 = _mm_unpackhi_epi16(*in0, *in1);
108 // r0 = [i0 i4 i1 i5 i2 i6 i3 i7]
109 // r1 = [iC i8 iD i9 iE iA iF iB]
110 const __m128i r2 = _mm_shuffle_epi32(r0, 0xB4);
111 const __m128i r3 = _mm_shuffle_epi32(r1, 0xB4);
112 // r2 = [i0 i4 i1 i5 i3 i7 i2 i6]
113 // r3 = [iC i8 iD i9 iF iB iE iA]
114
115 const __m128i t0 = _mm_add_epi16(r2, r3);
116 const __m128i t1 = _mm_sub_epi16(r2, r3);
117 // t0 = [a0 a4 a1 a5 a3 a7 a2 a6]
118 // t1 = [aC a8 aD a9 aF aB aE aA]
119
120 // Stage 2: multiply by constants (which gets us into 32 bits).
121 // The constants needed here are:
122 // k__cospi_A = [p16 p16 p16 p16 p16 m16 p16 m16]
123 // k__cospi_B = [p16 m16 p16 m16 p16 p16 p16 p16]
124 // k__cospi_C = [p08 p24 p08 p24 p24 m08 p24 m08]
125 // k__cospi_D = [p24 m08 p24 m08 p08 p24 p08 p24]
126 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_A);
127 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_B);
128 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_C);
129 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_D);
130 // Then add and right-shift to get back to 16-bit range
131 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
132 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
133 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
134 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
135 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
136 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
137 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
138 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
139 // w0 = [b0 b1 b7 b6]
140 // w1 = [b8 b9 bF bE]
141 // w2 = [b4 b5 b3 b2]
142 // w3 = [bC bD bB bA]
143 const __m128i x0 = _mm_packs_epi32(w0, w1);
144 const __m128i x1 = _mm_packs_epi32(w2, w3);
145
146 // x0 = [b0 b1 b7 b6 b8 b9 bF bE]
147 // x1 = [b4 b5 b3 b2 bC bD bB bA]
148 *in0 = _mm_shuffle_epi32(x0, 0xD8);
149 *in1 = _mm_shuffle_epi32(x1, 0x8D);
150 // in0 = [b0 b1 b8 b9 b7 b6 bF bE]
151 // in1 = [b3 b2 bB bA b4 b5 bC bD]
152 }
153 {
154 // vertical DCTs finished. Now we do the horizontal DCTs.
155 // Stage 3: Add/subtract
156
157 const __m128i t0 = ADD_EPI16(*in0, *in1);
158 const __m128i t1 = SUB_EPI16(*in0, *in1);
159
160 // Stage 4: multiply by constants (which gets us into 32 bits).
161 {
162 // The constants needed here are:
163 // k__cospi_E = [p16 p16 p16 p16 p16 p16 p16 p16]
164 // k__cospi_F = [p16 m16 p16 m16 p16 m16 p16 m16]
165 // k__cospi_G = [p08 p24 p08 p24 m08 m24 m08 m24]
166 // k__cospi_H = [p24 m08 p24 m08 m24 p08 m24 p08]
167 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_E);
168 const __m128i u1 = _mm_madd_epi16(t0, k__cospi_F);
169 const __m128i u2 = _mm_madd_epi16(t1, k__cospi_G);
170 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_H);
171 // Then add and right-shift to get back to 16-bit range
172 // but this combines the final right-shift as well to save operations
173 // This unusual rounding operations is to maintain bit-accurate
174 // compatibility with the c version of this function which has two
175 // rounding steps in a row.
176 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING2);
177 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING2);
178 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING2);
179 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING2);
180 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS2);
181 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS2);
182 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS2);
183 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS2);
184 *in0 = _mm_packs_epi32(w0, w2);
185 *in1 = _mm_packs_epi32(w1, w3);
186 }
187 }
188 }
189 #endif // defined(FDCT4x4_2D_HELPER)
190
191 #if defined(FDCT4x4_2D)
FDCT4x4_2D(const int16_t * input,tran_low_t * output,int stride)192 void FDCT4x4_2D(const int16_t *input, tran_low_t *output, int stride) {
193 // This 2D transform implements 4 vertical 1D transforms followed
194 // by 4 horizontal 1D transforms. The multiplies and adds are as given
195 // by Chen, Smith and Fralick ('77). The commands for moving the data
196 // around have been minimized by hand.
197 // For the purposes of the comments, the 16 inputs are referred to at i0
198 // through iF (in raster order), intermediate variables are a0, b0, c0
199 // through f, and correspond to the in-place computations mapped to input
200 // locations. The outputs, o0 through oF are labeled according to the
201 // output locations.
202 __m128i in0, in1;
203 FDCT4x4_2D_HELPER(input, stride, &in0, &in1);
204
205 // Post-condition (v + 1) >> 2 is now incorporated into previous
206 // add and right-shift commands. Only 2 store instructions needed
207 // because we are using the fact that 1/3 are stored just after 0/2.
208 storeu_output(&in0, output + 0 * 4);
209 storeu_output(&in1, output + 2 * 4);
210 }
211 #endif // defined(FDCT4x4_2D)
212
213 #if defined(FDCT4x4_2D_LP)
FDCT4x4_2D_LP(const int16_t * input,int16_t * output,int stride)214 void FDCT4x4_2D_LP(const int16_t *input, int16_t *output, int stride) {
215 __m128i in0, in1;
216 FDCT4x4_2D_HELPER(input, stride, &in0, &in1);
217 _mm_storeu_si128((__m128i *)(output + 0 * 4), in0);
218 _mm_storeu_si128((__m128i *)(output + 2 * 4), in1);
219 }
220 #endif // defined(FDCT4x4_2D_LP)
221
222 #if CONFIG_INTERNAL_STATS
FDCT8x8_2D(const int16_t * input,tran_low_t * output,int stride)223 void FDCT8x8_2D(const int16_t *input, tran_low_t *output, int stride) {
224 int pass;
225 // Constants
226 // When we use them, in one case, they are all the same. In all others
227 // it's a pair of them that we need to repeat four times. This is done
228 // by constructing the 32 bit constant corresponding to that pair.
229 const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64);
230 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
231 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
232 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
233 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
234 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
235 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
236 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
237 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
238 #if DCT_HIGH_BIT_DEPTH
239 int overflow;
240 #endif
241 // Load input
242 __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
243 __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
244 __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
245 __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
246 __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
247 __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
248 __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
249 __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
250 // Pre-condition input (shift by two)
251 in0 = _mm_slli_epi16(in0, 2);
252 in1 = _mm_slli_epi16(in1, 2);
253 in2 = _mm_slli_epi16(in2, 2);
254 in3 = _mm_slli_epi16(in3, 2);
255 in4 = _mm_slli_epi16(in4, 2);
256 in5 = _mm_slli_epi16(in5, 2);
257 in6 = _mm_slli_epi16(in6, 2);
258 in7 = _mm_slli_epi16(in7, 2);
259
260 // We do two passes, first the columns, then the rows. The results of the
261 // first pass are transposed so that the same column code can be reused. The
262 // results of the second pass are also transposed so that the rows (processed
263 // as columns) are put back in row positions.
264 for (pass = 0; pass < 2; pass++) {
265 // To store results of each pass before the transpose.
266 __m128i res0, res1, res2, res3, res4, res5, res6, res7;
267 // Add/subtract
268 const __m128i q0 = ADD_EPI16(in0, in7);
269 const __m128i q1 = ADD_EPI16(in1, in6);
270 const __m128i q2 = ADD_EPI16(in2, in5);
271 const __m128i q3 = ADD_EPI16(in3, in4);
272 const __m128i q4 = SUB_EPI16(in3, in4);
273 const __m128i q5 = SUB_EPI16(in2, in5);
274 const __m128i q6 = SUB_EPI16(in1, in6);
275 const __m128i q7 = SUB_EPI16(in0, in7);
276 #if DCT_HIGH_BIT_DEPTH
277 if (pass == 1) {
278 overflow =
279 check_epi16_overflow_x8(&q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7);
280 if (overflow) {
281 aom_highbd_fdct8x8_c(input, output, stride);
282 return;
283 }
284 }
285 #endif // DCT_HIGH_BIT_DEPTH
286 // Work on first four results
287 {
288 // Add/subtract
289 const __m128i r0 = ADD_EPI16(q0, q3);
290 const __m128i r1 = ADD_EPI16(q1, q2);
291 const __m128i r2 = SUB_EPI16(q1, q2);
292 const __m128i r3 = SUB_EPI16(q0, q3);
293 #if DCT_HIGH_BIT_DEPTH
294 overflow = check_epi16_overflow_x4(&r0, &r1, &r2, &r3);
295 if (overflow) {
296 aom_highbd_fdct8x8_c(input, output, stride);
297 return;
298 }
299 #endif // DCT_HIGH_BIT_DEPTH
300 // Interleave to do the multiply by constants which gets us into 32bits
301 {
302 const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
303 const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
304 const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
305 const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
306 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
307 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
308 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
309 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
310 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
311 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
312 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
313 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
314 // dct_const_round_shift
315 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
316 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
317 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
318 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
319 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
320 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
321 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
322 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
323 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
324 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
325 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
326 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
327 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
328 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
329 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
330 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
331 // Combine
332 res0 = _mm_packs_epi32(w0, w1);
333 res4 = _mm_packs_epi32(w2, w3);
334 res2 = _mm_packs_epi32(w4, w5);
335 res6 = _mm_packs_epi32(w6, w7);
336 #if DCT_HIGH_BIT_DEPTH
337 overflow = check_epi16_overflow_x4(&res0, &res4, &res2, &res6);
338 if (overflow) {
339 aom_highbd_fdct8x8_c(input, output, stride);
340 return;
341 }
342 #endif // DCT_HIGH_BIT_DEPTH
343 }
344 }
345 // Work on next four results
346 {
347 // Interleave to do the multiply by constants which gets us into 32bits
348 const __m128i d0 = _mm_unpacklo_epi16(q6, q5);
349 const __m128i d1 = _mm_unpackhi_epi16(q6, q5);
350 const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16);
351 const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16);
352 const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16);
353 const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16);
354 // dct_const_round_shift
355 const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING);
356 const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING);
357 const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING);
358 const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING);
359 const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS);
360 const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS);
361 const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS);
362 const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS);
363 // Combine
364 const __m128i r0 = _mm_packs_epi32(s0, s1);
365 const __m128i r1 = _mm_packs_epi32(s2, s3);
366 #if DCT_HIGH_BIT_DEPTH
367 overflow = check_epi16_overflow_x2(&r0, &r1);
368 if (overflow) {
369 aom_highbd_fdct8x8_c(input, output, stride);
370 return;
371 }
372 #endif // DCT_HIGH_BIT_DEPTH
373 {
374 // Add/subtract
375 const __m128i x0 = ADD_EPI16(q4, r0);
376 const __m128i x1 = SUB_EPI16(q4, r0);
377 const __m128i x2 = SUB_EPI16(q7, r1);
378 const __m128i x3 = ADD_EPI16(q7, r1);
379 #if DCT_HIGH_BIT_DEPTH
380 overflow = check_epi16_overflow_x4(&x0, &x1, &x2, &x3);
381 if (overflow) {
382 aom_highbd_fdct8x8_c(input, output, stride);
383 return;
384 }
385 #endif // DCT_HIGH_BIT_DEPTH
386 // Interleave to do the multiply by constants which gets us into 32bits
387 {
388 const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
389 const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
390 const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
391 const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
392 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
393 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
394 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
395 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
396 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
397 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
398 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
399 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
400 // dct_const_round_shift
401 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
402 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
403 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
404 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
405 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
406 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
407 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
408 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
409 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
410 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
411 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
412 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
413 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
414 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
415 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
416 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
417 // Combine
418 res1 = _mm_packs_epi32(w0, w1);
419 res7 = _mm_packs_epi32(w2, w3);
420 res5 = _mm_packs_epi32(w4, w5);
421 res3 = _mm_packs_epi32(w6, w7);
422 #if DCT_HIGH_BIT_DEPTH
423 overflow = check_epi16_overflow_x4(&res1, &res7, &res5, &res3);
424 if (overflow) {
425 aom_highbd_fdct8x8_c(input, output, stride);
426 return;
427 }
428 #endif // DCT_HIGH_BIT_DEPTH
429 }
430 }
431 }
432 // Transpose the 8x8.
433 {
434 // 00 01 02 03 04 05 06 07
435 // 10 11 12 13 14 15 16 17
436 // 20 21 22 23 24 25 26 27
437 // 30 31 32 33 34 35 36 37
438 // 40 41 42 43 44 45 46 47
439 // 50 51 52 53 54 55 56 57
440 // 60 61 62 63 64 65 66 67
441 // 70 71 72 73 74 75 76 77
442 const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
443 const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3);
444 const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1);
445 const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3);
446 const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5);
447 const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7);
448 const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5);
449 const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7);
450 // 00 10 01 11 02 12 03 13
451 // 20 30 21 31 22 32 23 33
452 // 04 14 05 15 06 16 07 17
453 // 24 34 25 35 26 36 27 37
454 // 40 50 41 51 42 52 43 53
455 // 60 70 61 71 62 72 63 73
456 // 54 54 55 55 56 56 57 57
457 // 64 74 65 75 66 76 67 77
458 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
459 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
460 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
461 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
462 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
463 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
464 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
465 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
466 // 00 10 20 30 01 11 21 31
467 // 40 50 60 70 41 51 61 71
468 // 02 12 22 32 03 13 23 33
469 // 42 52 62 72 43 53 63 73
470 // 04 14 24 34 05 15 21 36
471 // 44 54 64 74 45 55 61 76
472 // 06 16 26 36 07 17 27 37
473 // 46 56 66 76 47 57 67 77
474 in0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
475 in1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
476 in2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
477 in3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
478 in4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
479 in5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
480 in6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
481 in7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
482 // 00 10 20 30 40 50 60 70
483 // 01 11 21 31 41 51 61 71
484 // 02 12 22 32 42 52 62 72
485 // 03 13 23 33 43 53 63 73
486 // 04 14 24 34 44 54 64 74
487 // 05 15 25 35 45 55 65 75
488 // 06 16 26 36 46 56 66 76
489 // 07 17 27 37 47 57 67 77
490 }
491 }
492 // Post-condition output and store it
493 {
494 // Post-condition (division by two)
495 // division of two 16 bits signed numbers using shifts
496 // n / 2 = (n - (n >> 15)) >> 1
497 const __m128i sign_in0 = _mm_srai_epi16(in0, 15);
498 const __m128i sign_in1 = _mm_srai_epi16(in1, 15);
499 const __m128i sign_in2 = _mm_srai_epi16(in2, 15);
500 const __m128i sign_in3 = _mm_srai_epi16(in3, 15);
501 const __m128i sign_in4 = _mm_srai_epi16(in4, 15);
502 const __m128i sign_in5 = _mm_srai_epi16(in5, 15);
503 const __m128i sign_in6 = _mm_srai_epi16(in6, 15);
504 const __m128i sign_in7 = _mm_srai_epi16(in7, 15);
505 in0 = _mm_sub_epi16(in0, sign_in0);
506 in1 = _mm_sub_epi16(in1, sign_in1);
507 in2 = _mm_sub_epi16(in2, sign_in2);
508 in3 = _mm_sub_epi16(in3, sign_in3);
509 in4 = _mm_sub_epi16(in4, sign_in4);
510 in5 = _mm_sub_epi16(in5, sign_in5);
511 in6 = _mm_sub_epi16(in6, sign_in6);
512 in7 = _mm_sub_epi16(in7, sign_in7);
513 in0 = _mm_srai_epi16(in0, 1);
514 in1 = _mm_srai_epi16(in1, 1);
515 in2 = _mm_srai_epi16(in2, 1);
516 in3 = _mm_srai_epi16(in3, 1);
517 in4 = _mm_srai_epi16(in4, 1);
518 in5 = _mm_srai_epi16(in5, 1);
519 in6 = _mm_srai_epi16(in6, 1);
520 in7 = _mm_srai_epi16(in7, 1);
521 // store results
522 store_output(&in0, (output + 0 * 8));
523 store_output(&in1, (output + 1 * 8));
524 store_output(&in2, (output + 2 * 8));
525 store_output(&in3, (output + 3 * 8));
526 store_output(&in4, (output + 4 * 8));
527 store_output(&in5, (output + 5 * 8));
528 store_output(&in6, (output + 6 * 8));
529 store_output(&in7, (output + 7 * 8));
530 }
531 }
532 #endif // CONFIG_INTERNAL_STATS
533
534 #undef ADD_EPI16
535 #undef SUB_EPI16
536