1 // Auto-generated file. Do not edit!
2 // Template: src/f32-ibilinear-chw/sse.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include <xnnpack/ibilinear.h>
15
16
xnn_f32_ibilinear_chw_ukernel__sse_p4(size_t output_pixels,size_t channels,const float ** restrict input,size_t input_offset,const float * restrict weights,float * restrict output,size_t input_increment)17 void xnn_f32_ibilinear_chw_ukernel__sse_p4(
18 size_t output_pixels,
19 size_t channels,
20 const float**restrict input,
21 size_t input_offset,
22 const float*restrict weights,
23 float*restrict output,
24 size_t input_increment) XNN_OOB_READS
25 {
26 assert(output_pixels != 0);
27 assert(channels != 0);
28 assert(input_increment % sizeof(float) == 0);
29
30 do {
31 const float** i = input;
32 const float* w = weights;
33 size_t p = output_pixels;
34
35 for (; p >= 4; p -= 4) {
36 const float* itl0 = (const float*) ((uintptr_t) i[0] + input_offset);
37 const float* ibl0 = (const float*) ((uintptr_t) i[1] + input_offset);
38 const float* itl1 = (const float*) ((uintptr_t) i[2] + input_offset);
39 const float* ibl1 = (const float*) ((uintptr_t) i[3] + input_offset);
40 const float* itl2 = (const float*) ((uintptr_t) i[4] + input_offset);
41 const float* ibl2 = (const float*) ((uintptr_t) i[5] + input_offset);
42 const float* itl3 = (const float*) ((uintptr_t) i[6] + input_offset);
43 const float* ibl3 = (const float*) ((uintptr_t) i[7] + input_offset);
44 i += 8;
45
46 const __m128 vw0 = _mm_loadu_ps(w);
47 const __m128 vw1 = _mm_loadu_ps(w + 4);
48 w += 8;
49
50 const __m128 vtltr0 = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) itl0);
51 const __m128 vblbr0 = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) ibl0);
52 const __m128 vtltr2 = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) itl2);
53 const __m128 vblbr2 = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) ibl2);
54
55 const __m128 valphah = _mm_shuffle_ps(vw0, vw1, _MM_SHUFFLE(2, 0, 2, 0));
56 const __m128 valphav = _mm_shuffle_ps(vw0, vw1, _MM_SHUFFLE(3, 1, 3, 1));
57
58 const __m128 vtltr01 = _mm_loadh_pi(vtltr0, (const __m64*) itl1);
59 const __m128 vblbr01 = _mm_loadh_pi(vblbr0, (const __m64*) ibl1);
60 const __m128 vtltr23 = _mm_loadh_pi(vtltr2, (const __m64*) itl3);
61 const __m128 vblbr23 = _mm_loadh_pi(vblbr2, (const __m64*) ibl3);
62
63 const __m128 vldrd01 = _mm_sub_ps(vblbr01, vtltr01);
64 const __m128 vldrd23 = _mm_sub_ps(vblbr23, vtltr23);
65
66 const __m128 vld = _mm_shuffle_ps(vldrd01, vldrd23, _MM_SHUFFLE(2, 0, 2, 0));
67 const __m128 vrd = _mm_shuffle_ps(vldrd01, vldrd23, _MM_SHUFFLE(3, 1, 3, 1));
68
69 const __m128 vtl = _mm_shuffle_ps(vtltr01, vtltr23, _MM_SHUFFLE(2, 0, 2, 0));
70 const __m128 vtr = _mm_shuffle_ps(vtltr01, vtltr23, _MM_SHUFFLE(3, 1, 3, 1));
71
72 const __m128 vl = _mm_add_ps(vtl, _mm_mul_ps(vld, valphav));
73 const __m128 vr = _mm_add_ps(vtr, _mm_mul_ps(vrd, valphav));
74
75 const __m128 vd = _mm_sub_ps(vr, vl);
76 const __m128 vo = _mm_add_ps(vl, _mm_mul_ps(vd, valphah));
77
78 _mm_storeu_ps(output, vo);
79 output += 4;
80 }
81
82 if XNN_UNLIKELY(p != 0) {
83 if (p & 2) {
84 const __m128 vw = _mm_loadu_ps(w);
85 w += 4;
86
87 const __m128 valphah = _mm_shuffle_ps(vw, vw, _MM_SHUFFLE(2, 0, 2, 0));
88 const __m128 valphav = _mm_shuffle_ps(vw, vw, _MM_SHUFFLE(3, 1, 3, 1));
89
90 const float* itl0 = (const float*) ((uintptr_t) i[0] + input_offset);
91 const float* ibl0 = (const float*) ((uintptr_t) i[1] + input_offset);
92 const float* itl1 = (const float*) ((uintptr_t) i[2] + input_offset);
93 const float* ibl1 = (const float*) ((uintptr_t) i[3] + input_offset);
94 i += 4;
95
96 const __m128 vtltr = _mm_loadh_pi(_mm_loadl_pi(_mm_undefined_ps(), (const __m64*) itl0), (const __m64*) itl1);
97 const __m128 vblbr = _mm_loadh_pi(_mm_loadl_pi(_mm_undefined_ps(), (const __m64*) ibl0), (const __m64*) ibl1);
98
99 const __m128 vldrd = _mm_sub_ps(vblbr, vtltr);
100 const __m128 vld = _mm_shuffle_ps(vldrd, vldrd, _MM_SHUFFLE(2, 0, 2, 0));
101 const __m128 vrd = _mm_shuffle_ps(vldrd, vldrd, _MM_SHUFFLE(3, 1, 3, 1));
102
103 const __m128 vtl = _mm_shuffle_ps(vtltr, vtltr, _MM_SHUFFLE(2, 0, 2, 0));
104 const __m128 vtr = _mm_shuffle_ps(vtltr, vtltr, _MM_SHUFFLE(3, 1, 3, 1));
105
106 const __m128 vl = _mm_add_ps(vtl, _mm_mul_ps(vld, valphav));
107 const __m128 vr = _mm_add_ps(vtr, _mm_mul_ps(vrd, valphav));
108
109 const __m128 vd = _mm_sub_ps(vr, vl);
110 const __m128 vo = _mm_add_ps(vl, _mm_mul_ps(vd, valphah));
111
112 _mm_storel_pi((__m64*) output, vo);
113 output += 2;
114 }
115
116 if (p & 1) {
117 // We are computing the following formula:
118 // result = (1 - alpha_h) * (1 - alpha_v) * top_left +
119 // alpha_h * (1 - alpha_v) * top_right +
120 // (1 - alpha_h) * alpha_v * bottom_left +
121 // alpha_h * alpha_v * bottom_right.
122 //
123 // Rearranging gives
124 // result = left + alpha_h * (right - left),
125 // where
126 // left = top_left + alpha_v * (bottom_left - top_left),
127 // right = top_right + alpha_v * (bottom_right - top_right).
128
129 const float alphah = *w;
130 const __m128 valphav = _mm_load_ps1(w + 1);
131 w += 2;
132
133 const float* itl = (const float*) ((uintptr_t) i[0] + input_offset);
134 const float* ibl = (const float*) ((uintptr_t) i[1] + input_offset);
135 i += 2;
136
137 const __m128 vtltr = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) itl);
138 const __m128 vblbr = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) ibl);
139
140 // Compute at once
141 // left_diff = bottom_left - top_left
142 // right_diff = bottom_right - top_right
143 const __m128 vldrd = _mm_sub_ps(vblbr, vtltr);
144 const __m128 vlr = _mm_add_ps(vtltr, _mm_mul_ps(vldrd, valphav));
145
146 // Extract them and compute the result.
147 const float l = _mm_cvtss_f32(vlr);
148 const float r = _mm_cvtss_f32(_mm_shuffle_ps(vlr, vlr, 1));
149
150 *output++ = l + alphah * (r - l);
151 }
152 }
153
154 input_offset += input_increment;
155 } while (--channels != 0);
156 }
157