1 // Auto-generated file. Do not edit!
2 // Template: src/f32-f16-vcvt/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/common.h>
15 #include <xnnpack/vcvt.h>
16
17
xnn_f32_f16_vcvt_ukernel__neon_x8(size_t n,const float * input,void * output,const union xnn_f32_f16_cvt_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_f32_f16_vcvt_ukernel__neon_x8(
19 size_t n,
20 const float* input,
21 void* output,
22 const union xnn_f32_f16_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
23 {
24 assert(n != 0);
25 assert(n % sizeof(float) == 0);
26 assert(input != NULL);
27 assert(output != NULL);
28
29 const uint32x4_t vexp_bias = vld1q_dup_u32(¶ms->neon.exp_bias);
30 const float32x4_t vscale_to_inf = vld1q_dup_f32(¶ms->neon.scale_to_inf);
31 const uint32x4_t vexpw_max = vld1q_dup_u32(¶ms->neon.expw_max);
32 const float32x4_t vscale_to_zero = vld1q_dup_f32(¶ms->neon.scale_to_zero);
33 const uint32x4_t vbias_min = vdupq_n_u32(UINT32_C(0x40000000));
34 const uint16x8_t vexph_mask = vdupq_n_u16(UINT16_C(0x7C00));
35 const uint16x8_t vmanth_mask = vdupq_n_u16(UINT16_C(0x0FFF));
36 const uint16x8_t vsignh_mask = vdupq_n_u16(UINT16_C(0x8000));
37 const uint16x8_t vnanh = vdupq_n_u16(UINT16_C(0x7E00));
38
39 uint16_t* o = (uint16_t*) output;
40 for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
41 const float32x4_t vx0 = vld1q_f32(input); input += 4;
42 const float32x4_t vx1 = vld1q_f32(input); input += 4;
43
44 const float32x4_t vabsx0 = vabsq_f32(vx0);
45 const float32x4_t vabsx1 = vabsq_f32(vx1);
46
47 uint32x4_t vbias0 = vaddq_u32(vreinterpretq_u32_f32(vabsx0), vexp_bias);
48 uint32x4_t vbias1 = vaddq_u32(vreinterpretq_u32_f32(vabsx1), vexp_bias);
49
50 float32x4_t vf0 = vmulq_f32(vabsx0, vscale_to_inf);
51 float32x4_t vf1 = vmulq_f32(vabsx1, vscale_to_inf);
52 const uint32x4_t vnanmaskw0 = vcgtq_u32(vreinterpretq_u32_f32(vabsx0), vexpw_max);
53 const uint32x4_t vnanmaskw1 = vcgtq_u32(vreinterpretq_u32_f32(vabsx1), vexpw_max);
54
55 vbias0 = vandq_u32(vbias0, vexpw_max);
56 vbias1 = vandq_u32(vbias1, vexpw_max);
57 vf0 = vmulq_f32(vf0, vscale_to_zero);
58 vf1 = vmulq_f32(vf1, vscale_to_zero);
59
60 const uint16x8_t vnanmaskh0 = vcombine_u16(vmovn_u32(vnanmaskw0), vmovn_u32(vnanmaskw1));
61 vbias0 = vmaxq_u32(vbias0, vbias_min);
62 vbias1 = vmaxq_u32(vbias1, vbias_min);
63
64 vf0 = vaddq_f32(vf0, vreinterpretq_f32_u32(vbias0));
65 vf1 = vaddq_f32(vf1, vreinterpretq_f32_u32(vbias1));
66
67 uint16x8_t vexph0 = vcombine_u16(vshrn_n_u32(vreinterpretq_u32_f32(vf0), 13), vshrn_n_u32(vreinterpretq_u32_f32(vf1), 13));
68 uint16x8_t vmanth0 = vcombine_u16(vmovn_u32(vreinterpretq_u32_f32(vf0)), vmovn_u32(vreinterpretq_u32_f32(vf1)));
69 uint16x8_t vsignh0 = vcombine_u16(vshrn_n_u32(vreinterpretq_u32_f32(vx0), 16), vshrn_n_u32(vreinterpretq_u32_f32(vx1), 16));
70
71 vexph0 = vandq_u16(vexph0, vexph_mask);
72 vmanth0 = vandq_u16(vmanth0, vmanth_mask);
73 vsignh0 = vandq_u16(vsignh0, vsignh_mask);
74
75 uint16x8_t vh0 = vaddq_u16(vmanth0, vexph0);
76
77 vh0 = vbslq_u16(vnanmaskh0, vnanh, vh0);
78
79 vh0 = vorrq_u16(vh0, vsignh0);
80
81 vst1q_u16(o, vh0); o += 8;
82 }
83 for (; n >= 4 * sizeof(float); n -= 4 * sizeof(float)) {
84 const float32x4_t vx = vld1q_f32(input); input += 4;
85
86 const float32x4_t vabsx = vabsq_f32(vx);
87
88 uint32x4_t vbias = vaddq_u32(vreinterpretq_u32_f32(vabsx), vexp_bias);
89
90 float32x4_t vf = vmulq_f32(vabsx, vscale_to_inf);
91 const uint32x4_t vnanmaskw = vcgtq_u32(vreinterpretq_u32_f32(vabsx), vexpw_max);
92
93 vbias = vandq_u32(vbias, vexpw_max);
94 vf = vmulq_f32(vf, vscale_to_zero);
95
96 const uint16x4_t vnanmaskh = vmovn_u32(vnanmaskw);
97 vbias = vmaxq_u32(vbias, vbias_min);
98
99 vf = vaddq_f32(vf, vreinterpretq_f32_u32(vbias));
100
101 uint16x4_t vexph = vshrn_n_u32(vreinterpretq_u32_f32(vf), 13);
102 uint16x4_t vmanth = vmovn_u32(vreinterpretq_u32_f32(vf));
103 uint16x4_t vsignh = vshrn_n_u32(vreinterpretq_u32_f32(vx), 16);
104
105 vexph = vand_u16(vexph, vget_low_u16(vexph_mask));
106 vmanth = vand_u16(vmanth, vget_low_u16(vmanth_mask));
107 vsignh = vand_u16(vsignh, vget_low_u16(vsignh_mask));
108
109 uint16x4_t vh = vadd_u16(vmanth, vexph);
110
111 vh = vbsl_u16(vnanmaskh, vget_low_u16(vnanh), vh);
112
113 vh = vorr_u16(vh, vsignh);
114
115 vst1_u16(o, vh); o += 4;
116 }
117 if XNN_UNLIKELY(n != 0) {
118 assert(n % sizeof(float) == 0);
119 assert(n >= 1 * sizeof(float));
120 assert(n <= 3 * sizeof(float));
121 const float32x4_t vx = vld1q_f32(input);
122
123 const float32x4_t vabsx = vabsq_f32(vx);
124
125 uint32x4_t vbias = vaddq_u32(vreinterpretq_u32_f32(vabsx), vexp_bias);
126
127 float32x4_t vf = vmulq_f32(vabsx, vscale_to_inf);
128 const uint32x4_t vnanmaskw = vcgtq_u32(vreinterpretq_u32_f32(vabsx), vexpw_max);
129
130 vbias = vandq_u32(vbias, vexpw_max);
131 vf = vmulq_f32(vf, vscale_to_zero);
132
133 const uint16x4_t vnanmaskh = vmovn_u32(vnanmaskw);
134 vbias = vmaxq_u32(vbias, vbias_min);
135
136 vf = vaddq_f32(vf, vreinterpretq_f32_u32(vbias));
137
138 uint16x4_t vexph = vshrn_n_u32(vreinterpretq_u32_f32(vf), 13);
139 uint16x4_t vmanth = vmovn_u32(vreinterpretq_u32_f32(vf));
140 uint16x4_t vsignh = vshrn_n_u32(vreinterpretq_u32_f32(vx), 16);
141
142 vexph = vand_u16(vexph, vget_low_u16(vexph_mask));
143 vmanth = vand_u16(vmanth, vget_low_u16(vmanth_mask));
144 vsignh = vand_u16(vsignh, vget_low_u16(vsignh_mask));
145
146 uint16x4_t vh = vadd_u16(vmanth, vexph);
147
148 vh = vbsl_u16(vnanmaskh, vget_low_u16(vnanh), vh);
149
150 vh = vorr_u16(vh, vsignh);
151
152 if (n & (2 * sizeof(float))) {
153 vst1_lane_u32((void*) o, vreinterpret_u32_u16(vh), 0); o += 2;
154 vh = vext_u16(vh, vh, 2);
155 }
156 if (n & (1 * sizeof(float))) {
157 vst1_lane_u16(o, vh, 0);
158 }
159 }
160 }
161