1 /*
2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11 #ifndef AOM_AV1_ENCODER_X86_AV1_FWD_TXFM_SSE2_H_
12 #define AOM_AV1_ENCODER_X86_AV1_FWD_TXFM_SSE2_H_
13
14 #include <immintrin.h>
15
16 #include "config/aom_config.h"
17 #include "config/av1_rtcd.h"
18
19 #include "aom/aom_integer.h"
20 #include "aom_dsp/x86/transpose_sse2.h"
21 #include "aom_dsp/x86/txfm_common_sse2.h"
22
23 #ifdef __cplusplus
24 extern "C" {
25 #endif
26
27 void av1_fdct8x32_new_sse2(const __m128i *input, __m128i *output,
28 int8_t cos_bit);
29 void av1_fdct8x64_new_sse2(const __m128i *input, __m128i *output,
30 int8_t cos_bit);
31
fidentity4x4_new_sse2(const __m128i * const input,__m128i * const output,const int8_t cos_bit)32 static inline void fidentity4x4_new_sse2(const __m128i *const input,
33 __m128i *const output,
34 const int8_t cos_bit) {
35 (void)cos_bit;
36 const __m128i one = _mm_set1_epi16(1);
37
38 for (int i = 0; i < 4; ++i) {
39 const __m128i a = _mm_unpacklo_epi16(input[i], one);
40 const __m128i b = scale_round_sse2(a, NewSqrt2);
41 output[i] = _mm_packs_epi32(b, b);
42 }
43 }
44
fidentity8x4_new_sse2(const __m128i * const input,__m128i * const output,const int8_t cos_bit)45 static inline void fidentity8x4_new_sse2(const __m128i *const input,
46 __m128i *const output,
47 const int8_t cos_bit) {
48 (void)cos_bit;
49 const __m128i one = _mm_set1_epi16(1);
50
51 for (int i = 0; i < 4; ++i) {
52 const __m128i a_lo = _mm_unpacklo_epi16(input[i], one);
53 const __m128i a_hi = _mm_unpackhi_epi16(input[i], one);
54 const __m128i b_lo = scale_round_sse2(a_lo, NewSqrt2);
55 const __m128i b_hi = scale_round_sse2(a_hi, NewSqrt2);
56 output[i] = _mm_packs_epi32(b_lo, b_hi);
57 }
58 }
59
fidentity8x8_new_sse2(const __m128i * input,__m128i * output,int8_t cos_bit)60 static inline void fidentity8x8_new_sse2(const __m128i *input, __m128i *output,
61 int8_t cos_bit) {
62 (void)cos_bit;
63
64 output[0] = _mm_adds_epi16(input[0], input[0]);
65 output[1] = _mm_adds_epi16(input[1], input[1]);
66 output[2] = _mm_adds_epi16(input[2], input[2]);
67 output[3] = _mm_adds_epi16(input[3], input[3]);
68 output[4] = _mm_adds_epi16(input[4], input[4]);
69 output[5] = _mm_adds_epi16(input[5], input[5]);
70 output[6] = _mm_adds_epi16(input[6], input[6]);
71 output[7] = _mm_adds_epi16(input[7], input[7]);
72 }
73
fdct8x8_new_sse2(const __m128i * input,__m128i * output,int8_t cos_bit)74 static inline void fdct8x8_new_sse2(const __m128i *input, __m128i *output,
75 int8_t cos_bit) {
76 const int32_t *cospi = cospi_arr(cos_bit);
77 const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1));
78
79 const __m128i cospi_m32_p32 = pair_set_epi16(-cospi[32], cospi[32]);
80 const __m128i cospi_p32_p32 = pair_set_epi16(cospi[32], cospi[32]);
81 const __m128i cospi_p32_m32 = pair_set_epi16(cospi[32], -cospi[32]);
82 const __m128i cospi_p48_p16 = pair_set_epi16(cospi[48], cospi[16]);
83 const __m128i cospi_m16_p48 = pair_set_epi16(-cospi[16], cospi[48]);
84 const __m128i cospi_p56_p08 = pair_set_epi16(cospi[56], cospi[8]);
85 const __m128i cospi_m08_p56 = pair_set_epi16(-cospi[8], cospi[56]);
86 const __m128i cospi_p24_p40 = pair_set_epi16(cospi[24], cospi[40]);
87 const __m128i cospi_m40_p24 = pair_set_epi16(-cospi[40], cospi[24]);
88
89 // stage 1
90 __m128i x1[8];
91 x1[0] = _mm_adds_epi16(input[0], input[7]);
92 x1[7] = _mm_subs_epi16(input[0], input[7]);
93 x1[1] = _mm_adds_epi16(input[1], input[6]);
94 x1[6] = _mm_subs_epi16(input[1], input[6]);
95 x1[2] = _mm_adds_epi16(input[2], input[5]);
96 x1[5] = _mm_subs_epi16(input[2], input[5]);
97 x1[3] = _mm_adds_epi16(input[3], input[4]);
98 x1[4] = _mm_subs_epi16(input[3], input[4]);
99
100 // stage 2
101 __m128i x2[8];
102 x2[0] = _mm_adds_epi16(x1[0], x1[3]);
103 x2[3] = _mm_subs_epi16(x1[0], x1[3]);
104 x2[1] = _mm_adds_epi16(x1[1], x1[2]);
105 x2[2] = _mm_subs_epi16(x1[1], x1[2]);
106 x2[4] = x1[4];
107 btf_16_sse2(cospi_m32_p32, cospi_p32_p32, x1[5], x1[6], x2[5], x2[6]);
108 x2[7] = x1[7];
109
110 // stage 3
111 __m128i x3[8];
112 btf_16_sse2(cospi_p32_p32, cospi_p32_m32, x2[0], x2[1], x3[0], x3[1]);
113 btf_16_sse2(cospi_p48_p16, cospi_m16_p48, x2[2], x2[3], x3[2], x3[3]);
114 x3[4] = _mm_adds_epi16(x2[4], x2[5]);
115 x3[5] = _mm_subs_epi16(x2[4], x2[5]);
116 x3[6] = _mm_subs_epi16(x2[7], x2[6]);
117 x3[7] = _mm_adds_epi16(x2[7], x2[6]);
118
119 // stage 4 and 5
120 output[0] = x3[0];
121 output[4] = x3[1];
122 output[2] = x3[2];
123 output[6] = x3[3];
124 btf_16_sse2(cospi_p56_p08, cospi_m08_p56, x3[4], x3[7], output[1], output[7]);
125 btf_16_sse2(cospi_p24_p40, cospi_m40_p24, x3[5], x3[6], output[5], output[3]);
126 }
127
fadst8x8_new_sse2(const __m128i * input,__m128i * output,int8_t cos_bit)128 static inline void fadst8x8_new_sse2(const __m128i *input, __m128i *output,
129 int8_t cos_bit) {
130 const int32_t *cospi = cospi_arr(cos_bit);
131 const __m128i __zero = _mm_setzero_si128();
132 const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1));
133
134 const __m128i cospi_p32_p32 = pair_set_epi16(cospi[32], cospi[32]);
135 const __m128i cospi_p32_m32 = pair_set_epi16(cospi[32], -cospi[32]);
136 const __m128i cospi_p16_p48 = pair_set_epi16(cospi[16], cospi[48]);
137 const __m128i cospi_p48_m16 = pair_set_epi16(cospi[48], -cospi[16]);
138 const __m128i cospi_m48_p16 = pair_set_epi16(-cospi[48], cospi[16]);
139 const __m128i cospi_p04_p60 = pair_set_epi16(cospi[4], cospi[60]);
140 const __m128i cospi_p60_m04 = pair_set_epi16(cospi[60], -cospi[4]);
141 const __m128i cospi_p20_p44 = pair_set_epi16(cospi[20], cospi[44]);
142 const __m128i cospi_p44_m20 = pair_set_epi16(cospi[44], -cospi[20]);
143 const __m128i cospi_p36_p28 = pair_set_epi16(cospi[36], cospi[28]);
144 const __m128i cospi_p28_m36 = pair_set_epi16(cospi[28], -cospi[36]);
145 const __m128i cospi_p52_p12 = pair_set_epi16(cospi[52], cospi[12]);
146 const __m128i cospi_p12_m52 = pair_set_epi16(cospi[12], -cospi[52]);
147
148 // stage 1
149 __m128i x1[8];
150 x1[0] = input[0];
151 x1[1] = _mm_subs_epi16(__zero, input[7]);
152 x1[2] = _mm_subs_epi16(__zero, input[3]);
153 x1[3] = input[4];
154 x1[4] = _mm_subs_epi16(__zero, input[1]);
155 x1[5] = input[6];
156 x1[6] = input[2];
157 x1[7] = _mm_subs_epi16(__zero, input[5]);
158
159 // stage 2
160 __m128i x2[8];
161 x2[0] = x1[0];
162 x2[1] = x1[1];
163 btf_16_sse2(cospi_p32_p32, cospi_p32_m32, x1[2], x1[3], x2[2], x2[3]);
164 x2[4] = x1[4];
165 x2[5] = x1[5];
166 btf_16_sse2(cospi_p32_p32, cospi_p32_m32, x1[6], x1[7], x2[6], x2[7]);
167
168 // stage 3
169 __m128i x3[8];
170 x3[0] = _mm_adds_epi16(x2[0], x2[2]);
171 x3[2] = _mm_subs_epi16(x2[0], x2[2]);
172 x3[1] = _mm_adds_epi16(x2[1], x2[3]);
173 x3[3] = _mm_subs_epi16(x2[1], x2[3]);
174 x3[4] = _mm_adds_epi16(x2[4], x2[6]);
175 x3[6] = _mm_subs_epi16(x2[4], x2[6]);
176 x3[5] = _mm_adds_epi16(x2[5], x2[7]);
177 x3[7] = _mm_subs_epi16(x2[5], x2[7]);
178
179 // stage 4
180 __m128i x4[8];
181 x4[0] = x3[0];
182 x4[1] = x3[1];
183 x4[2] = x3[2];
184 x4[3] = x3[3];
185 btf_16_sse2(cospi_p16_p48, cospi_p48_m16, x3[4], x3[5], x4[4], x4[5]);
186 btf_16_sse2(cospi_m48_p16, cospi_p16_p48, x3[6], x3[7], x4[6], x4[7]);
187
188 // stage 5, 6 and 7
189 output[7] = _mm_adds_epi16(x4[0], x4[4]);
190 output[3] = _mm_subs_epi16(x4[0], x4[4]);
191 output[0] = _mm_adds_epi16(x4[1], x4[5]);
192 output[4] = _mm_subs_epi16(x4[1], x4[5]);
193 output[5] = _mm_adds_epi16(x4[2], x4[6]);
194 output[1] = _mm_subs_epi16(x4[2], x4[6]);
195 output[2] = _mm_adds_epi16(x4[3], x4[7]);
196 output[6] = _mm_subs_epi16(x4[3], x4[7]);
197
198 btf_16_sse2(cospi_p04_p60, cospi_p60_m04, output[7], output[0], output[7],
199 output[0]);
200 btf_16_sse2(cospi_p20_p44, cospi_p44_m20, output[5], output[2], output[5],
201 output[2]);
202 btf_16_sse2(cospi_p36_p28, cospi_p28_m36, output[3], output[4], output[3],
203 output[4]);
204 btf_16_sse2(cospi_p52_p12, cospi_p12_m52, output[1], output[6], output[1],
205 output[6]);
206 }
207
fidentity8x16_new_sse2(const __m128i * input,__m128i * output,int8_t cos_bit)208 static inline void fidentity8x16_new_sse2(const __m128i *input, __m128i *output,
209 int8_t cos_bit) {
210 (void)cos_bit;
211 const __m128i one = _mm_set1_epi16(1);
212
213 for (int i = 0; i < 16; ++i) {
214 const __m128i a_lo = _mm_unpacklo_epi16(input[i], one);
215 const __m128i a_hi = _mm_unpackhi_epi16(input[i], one);
216 const __m128i b_lo = scale_round_sse2(a_lo, 2 * NewSqrt2);
217 const __m128i b_hi = scale_round_sse2(a_hi, 2 * NewSqrt2);
218 output[i] = _mm_packs_epi32(b_lo, b_hi);
219 }
220 }
221
fidentity8x32_new_sse2(const __m128i * input,__m128i * output,int8_t cos_bit)222 static inline void fidentity8x32_new_sse2(const __m128i *input, __m128i *output,
223 int8_t cos_bit) {
224 (void)cos_bit;
225 for (int i = 0; i < 32; ++i) {
226 output[i] = _mm_slli_epi16(input[i], 2);
227 }
228 }
229
230 static const transform_1d_sse2 col_txfm8x32_arr[TX_TYPES] = {
231 av1_fdct8x32_new_sse2, // DCT_DCT
232 NULL, // ADST_DCT
233 NULL, // DCT_ADST
234 NULL, // ADST_ADST
235 NULL, // FLIPADST_DCT
236 NULL, // DCT_FLIPADST
237 NULL, // FLIPADST_FLIPADST
238 NULL, // ADST_FLIPADST
239 NULL, // FLIPADST_ADST
240 fidentity8x32_new_sse2, // IDTX
241 av1_fdct8x32_new_sse2, // V_DCT
242 fidentity8x32_new_sse2, // H_DCT
243 NULL, // V_ADST
244 NULL, // H_ADST
245 NULL, // V_FLIPADST
246 NULL // H_FLIPADST
247 };
248
249 #ifdef __cplusplus
250 }
251 #endif
252
253 #endif // AOM_AV1_ENCODER_X86_AV1_FWD_TXFM_SSE2_H_
254