1*4bdc9457SAndroid Build Coastguard Worker // Copyright (c) Facebook, Inc. and its affiliates.
2*4bdc9457SAndroid Build Coastguard Worker // All rights reserved.
3*4bdc9457SAndroid Build Coastguard Worker //
4*4bdc9457SAndroid Build Coastguard Worker // Copyright 2019 Google LLC
5*4bdc9457SAndroid Build Coastguard Worker //
6*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
7*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
8*4bdc9457SAndroid Build Coastguard Worker
9*4bdc9457SAndroid Build Coastguard Worker #include <emmintrin.h>
10*4bdc9457SAndroid Build Coastguard Worker
11*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/zip.h>
12*4bdc9457SAndroid Build Coastguard Worker
13*4bdc9457SAndroid Build Coastguard Worker
xnn_x8_zip_x4_ukernel__sse2(size_t n,const uint8_t * input,uint8_t * output)14*4bdc9457SAndroid Build Coastguard Worker void xnn_x8_zip_x4_ukernel__sse2(
15*4bdc9457SAndroid Build Coastguard Worker size_t n,
16*4bdc9457SAndroid Build Coastguard Worker const uint8_t* input,
17*4bdc9457SAndroid Build Coastguard Worker uint8_t* output)
18*4bdc9457SAndroid Build Coastguard Worker {
19*4bdc9457SAndroid Build Coastguard Worker const uint8_t* x = input;
20*4bdc9457SAndroid Build Coastguard Worker const uint8_t* y = (const uint8_t*) ((uintptr_t) x + n);
21*4bdc9457SAndroid Build Coastguard Worker const uint8_t* z = (const uint8_t*) ((uintptr_t) y + n);
22*4bdc9457SAndroid Build Coastguard Worker const uint8_t* w = (const uint8_t*) ((uintptr_t) z + n);
23*4bdc9457SAndroid Build Coastguard Worker uint8_t* o = output;
24*4bdc9457SAndroid Build Coastguard Worker
25*4bdc9457SAndroid Build Coastguard Worker if (n >= 16) {
26*4bdc9457SAndroid Build Coastguard Worker do {
27*4bdc9457SAndroid Build Coastguard Worker const __m128i vx = _mm_loadu_si128((const __m128i*) x);
28*4bdc9457SAndroid Build Coastguard Worker x += 16;
29*4bdc9457SAndroid Build Coastguard Worker const __m128i vy = _mm_loadu_si128((const __m128i*) y);
30*4bdc9457SAndroid Build Coastguard Worker y += 16;
31*4bdc9457SAndroid Build Coastguard Worker const __m128i vz = _mm_loadu_si128((const __m128i*) z);
32*4bdc9457SAndroid Build Coastguard Worker z += 16;
33*4bdc9457SAndroid Build Coastguard Worker const __m128i vw = _mm_loadu_si128((const __m128i*) w);
34*4bdc9457SAndroid Build Coastguard Worker w += 16;
35*4bdc9457SAndroid Build Coastguard Worker const __m128i vxy_lo = _mm_unpacklo_epi8(vx, vy);
36*4bdc9457SAndroid Build Coastguard Worker const __m128i vxy_hi = _mm_unpackhi_epi8(vx, vy);
37*4bdc9457SAndroid Build Coastguard Worker const __m128i vzw_lo = _mm_unpacklo_epi8(vz, vw);
38*4bdc9457SAndroid Build Coastguard Worker const __m128i vzw_hi = _mm_unpackhi_epi8(vz, vw);
39*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw0 = _mm_unpacklo_epi16(vxy_lo, vzw_lo);
40*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw1 = _mm_unpackhi_epi16(vxy_lo, vzw_lo);
41*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw2 = _mm_unpacklo_epi16(vxy_hi, vzw_hi);
42*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw3 = _mm_unpackhi_epi16(vxy_hi, vzw_hi);
43*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o, vxyzw0);
44*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o + 1, vxyzw1);
45*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o + 2, vxyzw2);
46*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o + 3, vxyzw3);
47*4bdc9457SAndroid Build Coastguard Worker o = (void*) ((uintptr_t) o + 64);
48*4bdc9457SAndroid Build Coastguard Worker n -= 16;
49*4bdc9457SAndroid Build Coastguard Worker } while (n >= 16);
50*4bdc9457SAndroid Build Coastguard Worker if (n != 0) {
51*4bdc9457SAndroid Build Coastguard Worker const size_t address_increment = n - 16;
52*4bdc9457SAndroid Build Coastguard Worker const __m128i vx = _mm_loadu_si128((const __m128i*) ((uintptr_t) x + address_increment));
53*4bdc9457SAndroid Build Coastguard Worker const __m128i vy = _mm_loadu_si128((const __m128i*) ((uintptr_t) y + address_increment));
54*4bdc9457SAndroid Build Coastguard Worker const __m128i vz = _mm_loadu_si128((const __m128i*) ((uintptr_t) z + address_increment));
55*4bdc9457SAndroid Build Coastguard Worker const __m128i vw = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + address_increment));
56*4bdc9457SAndroid Build Coastguard Worker const __m128i vxy_lo = _mm_unpacklo_epi8(vx, vy);
57*4bdc9457SAndroid Build Coastguard Worker const __m128i vxy_hi = _mm_unpackhi_epi8(vx, vy);
58*4bdc9457SAndroid Build Coastguard Worker const __m128i vzw_lo = _mm_unpacklo_epi8(vz, vw);
59*4bdc9457SAndroid Build Coastguard Worker const __m128i vzw_hi = _mm_unpackhi_epi8(vz, vw);
60*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw0 = _mm_unpacklo_epi16(vxy_lo, vzw_lo);
61*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw1 = _mm_unpackhi_epi16(vxy_lo, vzw_lo);
62*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw2 = _mm_unpacklo_epi16(vxy_hi, vzw_hi);
63*4bdc9457SAndroid Build Coastguard Worker const __m128i vxyzw3 = _mm_unpackhi_epi16(vxy_hi, vzw_hi);
64*4bdc9457SAndroid Build Coastguard Worker o = (void*) ((uintptr_t) o + address_increment * 4);
65*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o, vxyzw0);
66*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o + 1, vxyzw1);
67*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o + 2, vxyzw2);
68*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_si128((__m128i*) o + 3, vxyzw3);
69*4bdc9457SAndroid Build Coastguard Worker }
70*4bdc9457SAndroid Build Coastguard Worker } else {
71*4bdc9457SAndroid Build Coastguard Worker do {
72*4bdc9457SAndroid Build Coastguard Worker const uint8_t vx = *x++;
73*4bdc9457SAndroid Build Coastguard Worker const uint8_t vy = *y++;
74*4bdc9457SAndroid Build Coastguard Worker const uint8_t vz = *z++;
75*4bdc9457SAndroid Build Coastguard Worker const uint8_t vw = *w++;
76*4bdc9457SAndroid Build Coastguard Worker o[0] = vx;
77*4bdc9457SAndroid Build Coastguard Worker o[1] = vy;
78*4bdc9457SAndroid Build Coastguard Worker o[2] = vz;
79*4bdc9457SAndroid Build Coastguard Worker o[3] = vw;
80*4bdc9457SAndroid Build Coastguard Worker o += 4;
81*4bdc9457SAndroid Build Coastguard Worker } while (--n != 0);
82*4bdc9457SAndroid Build Coastguard Worker }
83*4bdc9457SAndroid Build Coastguard Worker }
84