1 // Copyright 2021 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7 #include <stddef.h>
8 #include <stdint.h>
9
10 #include <emmintrin.h>
11
12 #include <xnnpack/math-stubs.h>
13
14
xnn_math_f32_f16_cvt__sse2(size_t n,const float * input,void * output)15 void xnn_math_f32_f16_cvt__sse2(
16 size_t n,
17 const float* input,
18 void* output)
19 {
20 assert(n % (8 * sizeof(uint16_t)) == 0);
21
22 const __m128 vnonsign_mask = _mm_castsi128_ps(_mm_set1_epi32(0x7FFFFFFF));
23 const __m128i vexp_bias = _mm_set1_epi32(0x07800000);
24 const __m128 vscale_to_inf = _mm_set1_ps(0x1.0p+112f);
25 const __m128i vexpw_max = _mm_set1_epi32(0x7F800000);
26 const __m128 vscale_to_zero = _mm_set1_ps(0x1.0p-110f);
27 const __m128i vbias_min = _mm_set1_epi32(0x40008000);
28 const __m128i vmanth_mask = _mm_set1_epi32(0x0FFF);
29 const __m128i vexph_mask = _mm_set1_epi32(0x7C00);
30 const __m128i vnanh = _mm_set1_epi16(0x7E00);
31
32 uint16_t* o = (uint16_t*) output;
33 for (; n != 0; n -= 8 * sizeof(uint16_t)) {
34 const __m128 vx_lo = _mm_loadu_ps(input);
35 const __m128 vx_hi = _mm_loadu_ps(input + 4);
36 input += 8;
37
38 const __m128 vabsx_lo = _mm_and_ps(vx_lo, vnonsign_mask);
39 const __m128 vabsx_hi = _mm_and_ps(vx_hi, vnonsign_mask);
40
41 const __m128 vsignx_lo = _mm_xor_ps(vx_lo, vabsx_lo);
42 const __m128 vsignx_hi = _mm_xor_ps(vx_hi, vabsx_hi);
43 __m128i vbias_lo = _mm_add_epi32(_mm_castps_si128(vabsx_lo), vexp_bias);
44 __m128i vbias_hi = _mm_add_epi32(_mm_castps_si128(vabsx_hi), vexp_bias);
45 __m128 vf_lo = _mm_mul_ps(vabsx_lo, vscale_to_inf);
46 __m128 vf_hi = _mm_mul_ps(vabsx_hi, vscale_to_inf);
47 const __m128i vnanmaskw_lo = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_lo), vexpw_max);
48 const __m128i vnanmaskw_hi = _mm_cmpgt_epi32(_mm_castps_si128(vabsx_hi), vexpw_max);
49
50 vbias_lo = _mm_and_si128(vbias_lo, vexpw_max);
51 vbias_hi = _mm_and_si128(vbias_hi, vexpw_max);
52 vf_lo = _mm_mul_ps(vf_lo, vscale_to_zero);
53 vf_hi = _mm_mul_ps(vf_hi, vscale_to_zero);
54 const __m128i vnanmaskh = _mm_packs_epi32(vnanmaskw_lo, vnanmaskw_hi);
55 const __m128i vsignh = _mm_packs_epi32(_mm_castps_si128(vsignx_lo), _mm_castps_si128(vsignx_hi));
56
57 vbias_lo = _mm_max_epi16(vbias_lo, vbias_min);
58 vbias_hi = _mm_max_epi16(vbias_hi, vbias_min);
59 __m128i vh = _mm_and_si128(vnanh, vnanmaskh);
60
61 vf_lo = _mm_add_ps(vf_lo, _mm_castsi128_ps(vbias_lo));
62 vf_hi = _mm_add_ps(vf_hi, _mm_castsi128_ps(vbias_hi));
63 vh = _mm_or_si128(vh, vsignh);
64
65 __m128i vexpw_lo = _mm_srli_epi32(_mm_castps_si128(vf_lo), 13);
66 __m128i vexpw_hi = _mm_srli_epi32(_mm_castps_si128(vf_hi), 13);
67 const __m128i vmantw_lo = _mm_and_si128(_mm_castps_si128(vf_lo), vmanth_mask);
68 const __m128i vmantw_hi = _mm_and_si128(_mm_castps_si128(vf_hi), vmanth_mask);
69
70 vexpw_lo = _mm_and_si128(vexpw_lo, vexph_mask);
71 vexpw_hi = _mm_and_si128(vexpw_hi, vexph_mask);
72
73 const __m128i vnonsignw_lo = _mm_add_epi32(vmantw_lo, vexpw_lo);
74 const __m128i vnonsignw_hi = _mm_add_epi32(vmantw_hi, vexpw_hi);
75
76 const __m128i vnonsignh = _mm_packs_epi32(vnonsignw_lo, vnonsignw_hi);
77
78 vh = _mm_or_si128(vh, _mm_andnot_si128(vnanmaskh, vnonsignh));
79
80 _mm_storeu_si128((__m128i*) o, vh);
81 o += 8;
82 }
83 }
84