1 // Auto-generated file. Do not edit!
2 // Template: src/f32-f16-vcvt/sse.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <emmintrin.h>
13
14 #include <xnnpack/common.h>
15 #include <xnnpack/unaligned.h>
16 #include <xnnpack/vcvt.h>
17
18
xnn_f32_f16_vcvt_ukernel__sse2_x8(size_t n,const float * input,void * output,const union xnn_f32_f16_cvt_params params[restrict XNN_MIN_ELEMENTS (1)])19 void xnn_f32_f16_vcvt_ukernel__sse2_x8(
20 size_t n,
21 const float* input,
22 void* output,
23 const union xnn_f32_f16_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
24 {
25 assert(n != 0);
26 assert(n % sizeof(float) == 0);
27 assert(input != NULL);
28 assert(output != NULL);
29
30 const __m128 vnonsign_mask = _mm_load_ps((const float*) params->sse2.nonsign_mask);
31 const __m128i vexp_bias = _mm_load_si128((const __m128i*) params->sse2.exp_bias);
32 const __m128 vscale_to_inf = _mm_load_ps(params->sse2.scale_to_inf);
33 const __m128i vexpw_max = _mm_load_si128((const __m128i*) params->sse2.expw_max);
34 const __m128 vscale_to_zero = _mm_load_ps(params->sse2.scale_to_zero);
35 const __m128i vbias_min = _mm_load_si128((const __m128i*) params->sse2.bias_min);
36 const __m128i vmanth_mask = _mm_load_si128((const __m128i*) params->sse2.manth_mask);
37 const __m128i vexph_mask = _mm_load_si128((const __m128i*) params->sse2.exph_mask);
38 const __m128i vnanh = _mm_load_si128((const __m128i*) params->sse2.nanh);
39
40 uint16_t* o = (uint16_t*) output;
41 for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
42 const __m128 vx_lo = _mm_loadu_ps(input);
43 const __m128 vx_hi = _mm_loadu_ps(input + 4);
44 input += 8;
45
46 const __m128 vabsx_lo = _mm_and_ps(vx_lo, vnonsign_mask);
47 const __m128 vabsx_hi = _mm_and_ps(vx_hi, vnonsign_mask);
48
49 const __m128 vsignx_lo = _mm_xor_ps(vx_lo, vabsx_lo);
50 const __m128 vsignx_hi = _mm_xor_ps(vx_hi, vabsx_hi);
51 __m128i vbias_lo = _mm_add_epi32(_mm_castps_si128(vabsx_lo), vexp_bias);
52 __m128i vbias_hi = _mm_add_epi32(_mm_castps_si128(vabsx_hi), vexp_bias);
53 __m128 vf_lo = _mm_mul_ps(vabsx_lo, vscale_to_inf);
54 __m128 vf_hi = _mm_mul_ps(vabsx_hi, vscale_to_inf);
55 const __m128i vnanmaskw_lo = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_lo), vexpw_max);
56 const __m128i vnanmaskw_hi = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_hi), vexpw_max);
57
58 vbias_lo = _mm_and_si128(vbias_lo, vexpw_max);
59 vbias_hi = _mm_and_si128(vbias_hi, vexpw_max);
60 vf_lo = _mm_mul_ps(vf_lo, vscale_to_zero);
61 vf_hi = _mm_mul_ps(vf_hi, vscale_to_zero);
62 const __m128i vnanmaskh = _mm_packs_epi32(vnanmaskw_lo, vnanmaskw_hi);
63 const __m128i vsignh = _mm_packs_epi32(_mm_castps_si128(vsignx_lo), _mm_castps_si128(vsignx_hi));
64
65 vbias_lo = _mm_max_epi16(vbias_lo, vbias_min);
66 vbias_hi = _mm_max_epi16(vbias_hi, vbias_min);
67 __m128i vh = _mm_and_si128(vnanh, vnanmaskh);
68
69 vf_lo = _mm_add_ps(vf_lo, _mm_castsi128_ps(vbias_lo));
70 vf_hi = _mm_add_ps(vf_hi, _mm_castsi128_ps(vbias_hi));
71 vh = _mm_or_si128(vh, vsignh);
72
73 __m128i vexpw_lo = _mm_srli_epi32(_mm_castps_si128(vf_lo), 13);
74 __m128i vexpw_hi = _mm_srli_epi32(_mm_castps_si128(vf_hi), 13);
75 const __m128i vmantw_lo = _mm_and_si128(_mm_castps_si128(vf_lo), vmanth_mask);
76 const __m128i vmantw_hi = _mm_and_si128(_mm_castps_si128(vf_hi), vmanth_mask);
77
78 vexpw_lo = _mm_and_si128(vexpw_lo, vexph_mask);
79 vexpw_hi = _mm_and_si128(vexpw_hi, vexph_mask);
80
81 const __m128i vnonsignw_lo = _mm_add_epi32(vmantw_lo, vexpw_lo);
82 const __m128i vnonsignw_hi = _mm_add_epi32(vmantw_hi, vexpw_hi);
83
84 const __m128i vnonsignh = _mm_packs_epi32(vnonsignw_lo, vnonsignw_hi);
85
86 vh = _mm_or_si128(vh, _mm_andnot_si128(vnanmaskh, vnonsignh));
87
88 _mm_storeu_si128((__m128i*) o, vh);
89 o += 8;
90 }
91 if XNN_UNPREDICTABLE(n != 0) {
92 const __m128 vx_lo = _mm_loadu_ps(input);
93 const float* input_hi = (const float*) ((uintptr_t) input + (n & (4 * sizeof(float))));
94 const __m128 vx_hi = _mm_loadu_ps(input_hi);
95
96 const __m128 vabsx_lo = _mm_and_ps(vx_lo, vnonsign_mask);
97 const __m128 vabsx_hi = _mm_and_ps(vx_hi, vnonsign_mask);
98
99 const __m128 vsignx_lo = _mm_xor_ps(vx_lo, vabsx_lo);
100 const __m128 vsignx_hi = _mm_xor_ps(vx_hi, vabsx_hi);
101 __m128i vbias_lo = _mm_add_epi32(_mm_castps_si128(vabsx_lo), vexp_bias);
102 __m128i vbias_hi = _mm_add_epi32(_mm_castps_si128(vabsx_hi), vexp_bias);
103 __m128 vf_lo = _mm_mul_ps(vabsx_lo, vscale_to_inf);
104 __m128 vf_hi = _mm_mul_ps(vabsx_hi, vscale_to_inf);
105 const __m128i vnanmaskw_lo = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_lo), vexpw_max);
106 const __m128i vnanmaskw_hi = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_hi), vexpw_max);
107
108 vbias_lo = _mm_and_si128(vbias_lo, vexpw_max);
109 vbias_hi = _mm_and_si128(vbias_hi, vexpw_max);
110 vf_lo = _mm_mul_ps(vf_lo, vscale_to_zero);
111 vf_hi = _mm_mul_ps(vf_hi, vscale_to_zero);
112 const __m128i vnanmaskh = _mm_packs_epi32(vnanmaskw_lo, vnanmaskw_hi);
113 const __m128i vsignh = _mm_packs_epi32(_mm_castps_si128(vsignx_lo), _mm_castps_si128(vsignx_hi));
114
115 vbias_lo = _mm_max_epi16(vbias_lo, vbias_min);
116 vbias_hi = _mm_max_epi16(vbias_hi, vbias_min);
117 __m128i vh = _mm_and_si128(vnanh, vnanmaskh);
118
119 vf_lo = _mm_add_ps(vf_lo, _mm_castsi128_ps(vbias_lo));
120 vf_hi = _mm_add_ps(vf_hi, _mm_castsi128_ps(vbias_hi));
121 vh = _mm_or_si128(vh, vsignh);
122
123 __m128i vexpw_lo = _mm_srli_epi32(_mm_castps_si128(vf_lo), 13);
124 __m128i vexpw_hi = _mm_srli_epi32(_mm_castps_si128(vf_hi), 13);
125 const __m128i vmantw_lo = _mm_and_si128(_mm_castps_si128(vf_lo), vmanth_mask);
126 const __m128i vmantw_hi = _mm_and_si128(_mm_castps_si128(vf_hi), vmanth_mask);
127
128 vexpw_lo = _mm_and_si128(vexpw_lo, vexph_mask);
129 vexpw_hi = _mm_and_si128(vexpw_hi, vexph_mask);
130
131 const __m128i vnonsignw_lo = _mm_add_epi32(vmantw_lo, vexpw_lo);
132 const __m128i vnonsignw_hi = _mm_add_epi32(vmantw_hi, vexpw_hi);
133
134 const __m128i vnonsignh = _mm_packs_epi32(vnonsignw_lo, vnonsignw_hi);
135
136 vh = _mm_or_si128(vh, _mm_andnot_si128(vnanmaskh, vnonsignh));
137
138 if (n & (4 * sizeof(float))) {
139 _mm_storel_epi64((__m128i*) o, vh);
140 vh = _mm_unpackhi_epi64(vh, vh);
141 o += 4;
142 }
143 if (n & (2 * sizeof(float))) {
144 unaligned_store_u32(o, (uint32_t) _mm_cvtsi128_si32(vh));
145 vh = _mm_srli_epi64(vh, 32);
146 o += 2;
147 }
148 if (n & (1 * sizeof(float))) {
149 *o = (uint16_t) _mm_cvtsi128_si32(vh);
150 }
151 }
152 }
153