xref: /aosp_15_r20/external/XNNPACK/src/x16-transposec/gen/8x8-multi-switch-sse2.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/x32-transposec/sse2.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <immintrin.h>
11 
12 #include <assert.h>
13 
14 #include <xnnpack/common.h>
15 #include <xnnpack/math.h>
16 #include <xnnpack/transpose.h>
17 #include <xnnpack/unaligned.h>
18 
19 
xnn_x16_transposec_ukernel__8x8_multi_switch_sse2(const uint16_t * input,uint16_t * output,size_t input_stride,size_t output_stride,size_t block_width,size_t block_height)20 void xnn_x16_transposec_ukernel__8x8_multi_switch_sse2(
21     const uint16_t* input,
22     uint16_t* output,
23     size_t input_stride,
24     size_t output_stride,
25     size_t block_width,
26     size_t block_height) XNN_OOB_READS
27 {
28   assert(output_stride >= block_height * sizeof(uint16_t));
29   assert(input_stride >= block_width * sizeof(uint16_t));
30 
31   const size_t tile_height = 8;
32   const size_t tile_width = 8;
33   const size_t tile_hbytes = tile_height * sizeof(uint16_t);
34   const size_t tile_wbytes = tile_width * sizeof(uint16_t);
35   const size_t input_reset = tile_wbytes - round_down_po2(block_height, tile_height) * input_stride;
36   const size_t input_offset = tile_height * input_stride;
37   const size_t output_reset = tile_width * output_stride - round_down_po2(block_height, 2) * sizeof(uint16_t);
38 
39   const uint16_t* i0 = input;
40   const uint16_t* i1 = (const uint16_t*) ((uintptr_t) i0 + input_stride);
41   const uint16_t* i2 = (const uint16_t*) ((uintptr_t) i1 + input_stride);
42   const uint16_t* i3 = (const uint16_t*) ((uintptr_t) i2 + input_stride);
43   const uint16_t* i4 = (const uint16_t*) ((uintptr_t) i3 + input_stride);
44   const uint16_t* i5 = (const uint16_t*) ((uintptr_t) i4 + input_stride);
45   const uint16_t* i6 = (const uint16_t*) ((uintptr_t) i5 + input_stride);
46   const uint16_t* i7 = (const uint16_t*) ((uintptr_t) i6 + input_stride);
47   uint16_t* o = (uint16_t*) output;
48   const size_t minus_output_stride = -output_stride;
49 
50   do {
51     const size_t rem = min(block_width - 1, 7);
52     const size_t oN_stride = rem * output_stride;
53     size_t bh = block_height;
54     for (; bh >= 8; bh -= 8) {
55       const __m128i v3_0 = _mm_loadu_si128((const __m128i*) i0);
56       i0 = (uint16_t*) ((uintptr_t) i0 + input_offset);
57       const __m128i v3_1 = _mm_loadu_si128((const __m128i*) i1);
58       i1 = (uint16_t*) ((uintptr_t) i1 + input_offset);
59       const __m128i v3_2 = _mm_loadu_si128((const __m128i*) i2);
60       i2 = (uint16_t*) ((uintptr_t) i2 + input_offset);
61       const __m128i v3_3 = _mm_loadu_si128((const __m128i*) i3);
62       i3 = (uint16_t*) ((uintptr_t) i3 + input_offset);
63       const __m128i v3_4 = _mm_loadu_si128((const __m128i*) i4);
64       i4 = (uint16_t*) ((uintptr_t) i4 + input_offset);
65       const __m128i v3_5 = _mm_loadu_si128((const __m128i*) i5);
66       i5 = (uint16_t*) ((uintptr_t) i5 + input_offset);
67       const __m128i v3_6 = _mm_loadu_si128((const __m128i*) i6);
68       i6 = (uint16_t*) ((uintptr_t) i6 + input_offset);
69       const __m128i v3_7 = _mm_loadu_si128((const __m128i*) i7);
70       i7 = (uint16_t*) ((uintptr_t) i7 + input_offset);
71 
72       const __m128i v2_0 = _mm_unpacklo_epi16(v3_0, v3_1);
73       const __m128i v2_1 = _mm_unpackhi_epi16(v3_0, v3_1);
74       const __m128i v2_2 = _mm_unpacklo_epi16(v3_2, v3_3);
75       const __m128i v2_3 = _mm_unpackhi_epi16(v3_2, v3_3);
76       const __m128i v2_4 = _mm_unpacklo_epi16(v3_4, v3_5);
77       const __m128i v2_5 = _mm_unpackhi_epi16(v3_4, v3_5);
78       const __m128i v2_6 = _mm_unpacklo_epi16(v3_6, v3_7);
79       const __m128i v2_7 = _mm_unpackhi_epi16(v3_6, v3_7);
80 
81       const __m128i v1_0 = _mm_unpacklo_epi32(v2_0, v2_2);
82       const __m128i v1_1 = _mm_unpackhi_epi32(v2_0, v2_2);
83       const __m128i v1_2 = _mm_unpacklo_epi32(v2_1, v2_3);
84       const __m128i v1_3 = _mm_unpackhi_epi32(v2_1, v2_3);
85       const __m128i v1_4 = _mm_unpacklo_epi32(v2_4, v2_6);
86       const __m128i v1_5 = _mm_unpackhi_epi32(v2_4, v2_6);
87       const __m128i v1_6 = _mm_unpacklo_epi32(v2_5, v2_7);
88       const __m128i v1_7 = _mm_unpackhi_epi32(v2_5, v2_7);
89 
90       const __m128i v0_0 = _mm_unpacklo_epi64(v1_0, v1_4);
91       const __m128i v0_1 = _mm_unpackhi_epi64(v1_0, v1_4);
92       const __m128i v0_2 = _mm_unpacklo_epi64(v1_1, v1_5);
93       const __m128i v0_3 = _mm_unpackhi_epi64(v1_1, v1_5);
94       const __m128i v0_4 = _mm_unpacklo_epi64(v1_2, v1_6);
95       const __m128i v0_5 = _mm_unpackhi_epi64(v1_2, v1_6);
96       const __m128i v0_6 = _mm_unpacklo_epi64(v1_3, v1_7);
97       const __m128i v0_7 = _mm_unpackhi_epi64(v1_3, v1_7);
98 
99 
100       uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
101       switch (rem) {
102         case 7:
103           _mm_storeu_si128((__m128i*) oN, v0_7);
104           oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
105         case 6:
106           _mm_storeu_si128((__m128i*) oN, v0_6);
107           oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
108         case 5:
109           _mm_storeu_si128((__m128i*) oN, v0_5);
110           oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
111         case 4:
112           _mm_storeu_si128((__m128i*) oN, v0_4);
113           oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
114         case 3:
115           _mm_storeu_si128((__m128i*) oN, v0_3);
116           oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
117         case 2:
118           _mm_storeu_si128((__m128i*) oN, v0_2);
119           oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
120         case 1:
121           _mm_storeu_si128((__m128i*) oN, v0_1);
122         case 0:
123           _mm_storeu_si128((__m128i*) o, v0_0);
124           o = (uint16_t*) ((uintptr_t) o + tile_hbytes);
125           break;
126         default:
127           XNN_UNREACHABLE;
128       }
129     }
130     if (bh != 0) {
131       const __m128i v3_0 = _mm_loadu_si128((const __m128i*) i0);
132       if XNN_UNPREDICTABLE(bh < 2) {
133         i1 = i0;
134       }
135       const __m128i v3_1 = _mm_loadu_si128((const __m128i*) i1);
136       if XNN_UNPREDICTABLE(bh <= 2) {
137         i2 = i0;
138       }
139       const __m128i v3_2 = _mm_loadu_si128((const __m128i*) i2);
140       if XNN_UNPREDICTABLE(bh < 4) {
141         i3 = i0;
142       }
143       const __m128i v3_3 = _mm_loadu_si128((const __m128i*) i3);
144       if XNN_UNPREDICTABLE(bh <= 4) {
145         i4 = i0;
146       }
147       const __m128i v3_4 = _mm_loadu_si128((const __m128i*) i4);
148       if XNN_UNPREDICTABLE(bh < 6) {
149         i5 = i0;
150       }
151       const __m128i v3_5 = _mm_loadu_si128((const __m128i*) i5);
152       if XNN_UNPREDICTABLE(bh <= 6) {
153         i6 = i0;
154       }
155       const __m128i v3_6 = _mm_loadu_si128((const __m128i*) i6);
156       const __m128i v3_7 = _mm_undefined_si128();
157 
158       const __m128i v2_0 = _mm_unpacklo_epi16(v3_0, v3_1);
159       const __m128i v2_1 = _mm_unpackhi_epi16(v3_0, v3_1);
160       const __m128i v2_2 = _mm_unpacklo_epi16(v3_2, v3_3);
161       const __m128i v2_3 = _mm_unpackhi_epi16(v3_2, v3_3);
162       const __m128i v2_4 = _mm_unpacklo_epi16(v3_4, v3_5);
163       const __m128i v2_5 = _mm_unpackhi_epi16(v3_4, v3_5);
164       const __m128i v2_6 = _mm_unpacklo_epi16(v3_6, v3_7);
165       const __m128i v2_7 = _mm_unpackhi_epi16(v3_6, v3_7);
166 
167       const __m128i v1_0 = _mm_unpacklo_epi32(v2_0, v2_2);
168       const __m128i v1_1 = _mm_unpackhi_epi32(v2_0, v2_2);
169       const __m128i v1_2 = _mm_unpacklo_epi32(v2_1, v2_3);
170       const __m128i v1_3 = _mm_unpackhi_epi32(v2_1, v2_3);
171       const __m128i v1_4 = _mm_unpacklo_epi32(v2_4, v2_6);
172       const __m128i v1_5 = _mm_unpackhi_epi32(v2_4, v2_6);
173       const __m128i v1_6 = _mm_unpacklo_epi32(v2_5, v2_7);
174       const __m128i v1_7 = _mm_unpackhi_epi32(v2_5, v2_7);
175 
176       __m128i v0_0 = _mm_unpacklo_epi64(v1_0, v1_4);
177       __m128i v0_1 = _mm_unpackhi_epi64(v1_0, v1_4);
178       __m128i v0_2 = _mm_unpacklo_epi64(v1_1, v1_5);
179       __m128i v0_3 = _mm_unpackhi_epi64(v1_1, v1_5);
180       __m128i v0_4 = _mm_unpacklo_epi64(v1_2, v1_6);
181       __m128i v0_5 = _mm_unpackhi_epi64(v1_2, v1_6);
182       __m128i v0_6 = _mm_unpacklo_epi64(v1_3, v1_7);
183       __m128i v0_7 = _mm_unpackhi_epi64(v1_3, v1_7);
184 
185 
186       if (bh & 4) {
187         uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
188         switch (rem) {
189           case 7:
190             _mm_storel_epi64((__m128i*) oN, v0_7);
191             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
192           case 6:
193             _mm_storel_epi64((__m128i*) oN, v0_6);
194             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
195           case 5:
196             _mm_storel_epi64((__m128i*) oN, v0_5);
197             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
198           case 4:
199             _mm_storel_epi64((__m128i*) oN, v0_4);
200             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
201           case 3:
202             _mm_storel_epi64((__m128i*) oN, v0_3);
203             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
204           case 2:
205             _mm_storel_epi64((__m128i*) oN, v0_2);
206             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
207           case 1:
208             _mm_storel_epi64((__m128i*) oN, v0_1);
209           case 0:
210             _mm_storel_epi64((__m128i*) o, v0_0);
211             break;
212           default:
213             XNN_UNREACHABLE;
214         }
215         o += 4;
216         v0_0 = _mm_unpackhi_epi64(v0_0, v0_0);
217         v0_1 = _mm_unpackhi_epi64(v0_1, v0_1);
218         v0_2 = _mm_unpackhi_epi64(v0_2, v0_2);
219         v0_3 = _mm_unpackhi_epi64(v0_3, v0_3);
220         v0_4 = _mm_unpackhi_epi64(v0_4, v0_4);
221         v0_5 = _mm_unpackhi_epi64(v0_5, v0_5);
222         v0_6 = _mm_unpackhi_epi64(v0_6, v0_6);
223         v0_7 = _mm_unpackhi_epi64(v0_7, v0_7);
224       }
225 
226       if (bh & 2) {
227         uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
228         switch (rem) {
229           case 7:
230             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_7));
231             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
232           case 6:
233             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_6));
234             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
235           case 5:
236             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_5));
237             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
238           case 4:
239             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_4));
240             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
241           case 3:
242             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_3));
243             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
244           case 2:
245             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_2));
246             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
247           case 1:
248             unaligned_store_u32(oN, (uint32_t) _mm_cvtsi128_si32(v0_1));
249           case 0:
250             unaligned_store_u32(o, (uint32_t) _mm_cvtsi128_si32(v0_0));
251             break;
252           default:
253             XNN_UNREACHABLE;
254         }
255         o += 2;
256         v0_0 = _mm_srli_epi64(v0_0, 32);
257         v0_1 = _mm_srli_epi64(v0_1, 32);
258         v0_2 = _mm_srli_epi64(v0_2, 32);
259         v0_3 = _mm_srli_epi64(v0_3, 32);
260         v0_4 = _mm_srli_epi64(v0_4, 32);
261         v0_5 = _mm_srli_epi64(v0_5, 32);
262         v0_6 = _mm_srli_epi64(v0_6, 32);
263         v0_7 = _mm_srli_epi64(v0_7, 32);
264       }
265       if (bh & 1) {
266         uint16_t* oN = (uint16_t*) ((uintptr_t) o + oN_stride);
267         switch (rem) {
268           case 7:
269             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_7));
270             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
271           case 6:
272             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_6));
273             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
274           case 5:
275             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_5));
276             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
277           case 4:
278             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_4));
279             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
280           case 3:
281             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_3));
282             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
283           case 2:
284             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_2));
285             oN = (uint16_t*) ((uintptr_t) oN + minus_output_stride);
286           case 1:
287             unaligned_store_u16(oN, (uint16_t) _mm_cvtsi128_si32(v0_1));
288           case 0:
289             unaligned_store_u16(o, (uint16_t) _mm_cvtsi128_si32(v0_0));
290             break;
291           default:
292             XNN_UNREACHABLE;
293         }
294       }
295     }
296 
297     i0 = (const uint16_t*) ((uintptr_t) i0 + input_reset);
298     i1 = (const uint16_t*) ((uintptr_t) i0 + input_stride);
299     i2 = (const uint16_t*) ((uintptr_t) i1 + input_stride);
300     i3 = (const uint16_t*) ((uintptr_t) i2 + input_stride);
301     i4 = (const uint16_t*) ((uintptr_t) i3 + input_stride);
302     i5 = (const uint16_t*) ((uintptr_t) i4 + input_stride);
303     i6 = (const uint16_t*) ((uintptr_t) i5 + input_stride);
304     i7 = (const uint16_t*) ((uintptr_t) i6 + input_stride);
305     o = (uint16_t*) ((uintptr_t) o + output_reset);
306     block_width = doz(block_width, tile_width);
307   } while (block_width != 0);
308 }
309