xref: /aosp_15_r20/external/XNNPACK/src/f32-vsqrt/gen/neonfma-nr2fma1adj-x28.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-vsqrt/neonfma-nr2fma1adj.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 #include <math.h>
12 
13 #include <arm_neon.h>
14 
15 #include <xnnpack/common.h>
16 #include <xnnpack/vunary.h>
17 
18 
xnn_f32_vsqrt_ukernel__neonfma_nr2fma1adj_x28(size_t n,const float * x,float * y,const union xnn_f32_sqrt_params params[restrict XNN_MIN_ELEMENTS (1)])19 void xnn_f32_vsqrt_ukernel__neonfma_nr2fma1adj_x28(
20     size_t n,
21     const float* x,
22     float* y,
23     const union xnn_f32_sqrt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
24 {
25   assert(n != 0);
26   assert(n % sizeof(float) == 0);
27 
28   const float32x4_t vhalf = vmovq_n_f32(0.5f);
29   for (; n >= 28 * sizeof(float); n -= 28 * sizeof(float)) {
30     const float32x4_t vx0123 = vld1q_f32(x); x += 4;
31     const float32x4_t vx4567 = vld1q_f32(x); x += 4;
32     const float32x4_t vx89AB = vld1q_f32(x); x += 4;
33     const float32x4_t vxCDEF = vld1q_f32(x); x += 4;
34     const float32x4_t vxGHIJ = vld1q_f32(x); x += 4;
35     const float32x4_t vxKLMN = vld1q_f32(x); x += 4;
36     const float32x4_t vxOPQR = vld1q_f32(x); x += 4;
37 
38     const float32x4_t vrsqrtx0123 = vrsqrteq_f32(vx0123);
39     const float32x4_t vrsqrtx4567 = vrsqrteq_f32(vx4567);
40     const float32x4_t vrsqrtx89AB = vrsqrteq_f32(vx89AB);
41     const float32x4_t vrsqrtxCDEF = vrsqrteq_f32(vxCDEF);
42     const float32x4_t vrsqrtxGHIJ = vrsqrteq_f32(vxGHIJ);
43     const float32x4_t vrsqrtxKLMN = vrsqrteq_f32(vxKLMN);
44     const float32x4_t vrsqrtxOPQR = vrsqrteq_f32(vxOPQR);
45 
46     float32x4_t vsqrtx0123 = vmulq_f32(vrsqrtx0123, vx0123);
47     float32x4_t vhalfrsqrtx0123 = vmulq_f32(vrsqrtx0123, vhalf);
48     float32x4_t vsqrtx4567 = vmulq_f32(vrsqrtx4567, vx4567);
49     float32x4_t vhalfrsqrtx4567 = vmulq_f32(vrsqrtx4567, vhalf);
50     float32x4_t vsqrtx89AB = vmulq_f32(vrsqrtx89AB, vx89AB);
51     float32x4_t vhalfrsqrtx89AB = vmulq_f32(vrsqrtx89AB, vhalf);
52     float32x4_t vsqrtxCDEF = vmulq_f32(vrsqrtxCDEF, vxCDEF);
53     float32x4_t vhalfrsqrtxCDEF = vmulq_f32(vrsqrtxCDEF, vhalf);
54     float32x4_t vsqrtxGHIJ = vmulq_f32(vrsqrtxGHIJ, vxGHIJ);
55     float32x4_t vhalfrsqrtxGHIJ = vmulq_f32(vrsqrtxGHIJ, vhalf);
56     float32x4_t vsqrtxKLMN = vmulq_f32(vrsqrtxKLMN, vxKLMN);
57     float32x4_t vhalfrsqrtxKLMN = vmulq_f32(vrsqrtxKLMN, vhalf);
58     float32x4_t vsqrtxOPQR = vmulq_f32(vrsqrtxOPQR, vxOPQR);
59     float32x4_t vhalfrsqrtxOPQR = vmulq_f32(vrsqrtxOPQR, vhalf);
60 
61     float32x4_t vresidual0123 = vfmsq_f32(vhalf, vsqrtx0123, vhalfrsqrtx0123);
62     float32x4_t vresidual4567 = vfmsq_f32(vhalf, vsqrtx4567, vhalfrsqrtx4567);
63     float32x4_t vresidual89AB = vfmsq_f32(vhalf, vsqrtx89AB, vhalfrsqrtx89AB);
64     float32x4_t vresidualCDEF = vfmsq_f32(vhalf, vsqrtxCDEF, vhalfrsqrtxCDEF);
65     float32x4_t vresidualGHIJ = vfmsq_f32(vhalf, vsqrtxGHIJ, vhalfrsqrtxGHIJ);
66     float32x4_t vresidualKLMN = vfmsq_f32(vhalf, vsqrtxKLMN, vhalfrsqrtxKLMN);
67     float32x4_t vresidualOPQR = vfmsq_f32(vhalf, vsqrtxOPQR, vhalfrsqrtxOPQR);
68 
69     vhalfrsqrtx0123 = vfmaq_f32(vhalfrsqrtx0123, vresidual0123, vhalfrsqrtx0123);
70     vsqrtx0123 = vfmaq_f32(vsqrtx0123, vresidual0123, vsqrtx0123);
71     vhalfrsqrtx4567 = vfmaq_f32(vhalfrsqrtx4567, vresidual4567, vhalfrsqrtx4567);
72     vsqrtx4567 = vfmaq_f32(vsqrtx4567, vresidual4567, vsqrtx4567);
73     vhalfrsqrtx89AB = vfmaq_f32(vhalfrsqrtx89AB, vresidual89AB, vhalfrsqrtx89AB);
74     vsqrtx89AB = vfmaq_f32(vsqrtx89AB, vresidual89AB, vsqrtx89AB);
75     vhalfrsqrtxCDEF = vfmaq_f32(vhalfrsqrtxCDEF, vresidualCDEF, vhalfrsqrtxCDEF);
76     vsqrtxCDEF = vfmaq_f32(vsqrtxCDEF, vresidualCDEF, vsqrtxCDEF);
77     vhalfrsqrtxGHIJ = vfmaq_f32(vhalfrsqrtxGHIJ, vresidualGHIJ, vhalfrsqrtxGHIJ);
78     vsqrtxGHIJ = vfmaq_f32(vsqrtxGHIJ, vresidualGHIJ, vsqrtxGHIJ);
79     vhalfrsqrtxKLMN = vfmaq_f32(vhalfrsqrtxKLMN, vresidualKLMN, vhalfrsqrtxKLMN);
80     vsqrtxKLMN = vfmaq_f32(vsqrtxKLMN, vresidualKLMN, vsqrtxKLMN);
81     vhalfrsqrtxOPQR = vfmaq_f32(vhalfrsqrtxOPQR, vresidualOPQR, vhalfrsqrtxOPQR);
82     vsqrtxOPQR = vfmaq_f32(vsqrtxOPQR, vresidualOPQR, vsqrtxOPQR);
83 
84     vresidual0123 = vfmsq_f32(vhalf, vsqrtx0123, vhalfrsqrtx0123);
85     vresidual4567 = vfmsq_f32(vhalf, vsqrtx4567, vhalfrsqrtx4567);
86     vresidual89AB = vfmsq_f32(vhalf, vsqrtx89AB, vhalfrsqrtx89AB);
87     vresidualCDEF = vfmsq_f32(vhalf, vsqrtxCDEF, vhalfrsqrtxCDEF);
88     vresidualGHIJ = vfmsq_f32(vhalf, vsqrtxGHIJ, vhalfrsqrtxGHIJ);
89     vresidualKLMN = vfmsq_f32(vhalf, vsqrtxKLMN, vhalfrsqrtxKLMN);
90     vresidualOPQR = vfmsq_f32(vhalf, vsqrtxOPQR, vhalfrsqrtxOPQR);
91 
92     vhalfrsqrtx0123 = vfmaq_f32(vhalfrsqrtx0123, vresidual0123, vhalfrsqrtx0123);
93     vsqrtx0123 = vfmaq_f32(vsqrtx0123, vresidual0123, vsqrtx0123);
94     vhalfrsqrtx4567 = vfmaq_f32(vhalfrsqrtx4567, vresidual4567, vhalfrsqrtx4567);
95     vsqrtx4567 = vfmaq_f32(vsqrtx4567, vresidual4567, vsqrtx4567);
96     vhalfrsqrtx89AB = vfmaq_f32(vhalfrsqrtx89AB, vresidual89AB, vhalfrsqrtx89AB);
97     vsqrtx89AB = vfmaq_f32(vsqrtx89AB, vresidual89AB, vsqrtx89AB);
98     vhalfrsqrtxCDEF = vfmaq_f32(vhalfrsqrtxCDEF, vresidualCDEF, vhalfrsqrtxCDEF);
99     vsqrtxCDEF = vfmaq_f32(vsqrtxCDEF, vresidualCDEF, vsqrtxCDEF);
100     vhalfrsqrtxGHIJ = vfmaq_f32(vhalfrsqrtxGHIJ, vresidualGHIJ, vhalfrsqrtxGHIJ);
101     vsqrtxGHIJ = vfmaq_f32(vsqrtxGHIJ, vresidualGHIJ, vsqrtxGHIJ);
102     vhalfrsqrtxKLMN = vfmaq_f32(vhalfrsqrtxKLMN, vresidualKLMN, vhalfrsqrtxKLMN);
103     vsqrtxKLMN = vfmaq_f32(vsqrtxKLMN, vresidualKLMN, vsqrtxKLMN);
104     vhalfrsqrtxOPQR = vfmaq_f32(vhalfrsqrtxOPQR, vresidualOPQR, vhalfrsqrtxOPQR);
105     vsqrtxOPQR = vfmaq_f32(vsqrtxOPQR, vresidualOPQR, vsqrtxOPQR);
106 
107     const float32x4_t vadjustment0123 = vfmsq_f32(vx0123, vsqrtx0123, vsqrtx0123);
108     const float32x4_t vadjustment4567 = vfmsq_f32(vx4567, vsqrtx4567, vsqrtx4567);
109     const float32x4_t vadjustment89AB = vfmsq_f32(vx89AB, vsqrtx89AB, vsqrtx89AB);
110     const float32x4_t vadjustmentCDEF = vfmsq_f32(vxCDEF, vsqrtxCDEF, vsqrtxCDEF);
111     const float32x4_t vadjustmentGHIJ = vfmsq_f32(vxGHIJ, vsqrtxGHIJ, vsqrtxGHIJ);
112     const float32x4_t vadjustmentKLMN = vfmsq_f32(vxKLMN, vsqrtxKLMN, vsqrtxKLMN);
113     const float32x4_t vadjustmentOPQR = vfmsq_f32(vxOPQR, vsqrtxOPQR, vsqrtxOPQR);
114 
115     const float32x4_t vy0123 = vfmaq_f32(vsqrtx0123, vhalfrsqrtx0123, vadjustment0123);
116     const float32x4_t vy4567 = vfmaq_f32(vsqrtx4567, vhalfrsqrtx4567, vadjustment4567);
117     const float32x4_t vy89AB = vfmaq_f32(vsqrtx89AB, vhalfrsqrtx89AB, vadjustment89AB);
118     const float32x4_t vyCDEF = vfmaq_f32(vsqrtxCDEF, vhalfrsqrtxCDEF, vadjustmentCDEF);
119     const float32x4_t vyGHIJ = vfmaq_f32(vsqrtxGHIJ, vhalfrsqrtxGHIJ, vadjustmentGHIJ);
120     const float32x4_t vyKLMN = vfmaq_f32(vsqrtxKLMN, vhalfrsqrtxKLMN, vadjustmentKLMN);
121     const float32x4_t vyOPQR = vfmaq_f32(vsqrtxOPQR, vhalfrsqrtxOPQR, vadjustmentOPQR);
122 
123     vst1q_f32(y, vy0123); y += 4;
124     vst1q_f32(y, vy4567); y += 4;
125     vst1q_f32(y, vy89AB); y += 4;
126     vst1q_f32(y, vyCDEF); y += 4;
127     vst1q_f32(y, vyGHIJ); y += 4;
128     vst1q_f32(y, vyKLMN); y += 4;
129     vst1q_f32(y, vyOPQR); y += 4;
130   }
131   for (; n >= 4 * sizeof(float); n -= 4 * sizeof(float)) {
132     const float32x4_t vx = vld1q_f32(x); x += 4;
133     const float32x4_t vrsqrtx = vrsqrteq_f32(vx);
134     float32x4_t vsqrtx = vmulq_f32(vrsqrtx, vx);
135     float32x4_t vhalfrsqrtx = vmulq_f32(vrsqrtx, vhalf);
136     float32x4_t vresidual = vfmsq_f32(vhalf, vsqrtx, vhalfrsqrtx);
137     vhalfrsqrtx = vfmaq_f32(vhalfrsqrtx, vresidual, vhalfrsqrtx);
138     vsqrtx = vfmaq_f32(vsqrtx, vresidual, vsqrtx);
139     vresidual = vfmsq_f32(vhalf, vsqrtx, vhalfrsqrtx);
140     vhalfrsqrtx = vfmaq_f32(vhalfrsqrtx, vresidual, vhalfrsqrtx);
141     vsqrtx = vfmaq_f32(vsqrtx, vresidual, vsqrtx);
142     const float32x4_t vadjustment = vfmsq_f32(vx, vsqrtx, vsqrtx);
143     const float32x4_t vy = vfmaq_f32(vsqrtx, vhalfrsqrtx, vadjustment);
144     vst1q_f32(y, vy); y += 4;
145   }
146   if XNN_UNLIKELY(n != 0) {
147     const float32x4_t vx = vld1q_f32(x);
148     const float32x4_t vrsqrtx = vrsqrteq_f32(vx);
149     float32x4_t vsqrtx = vmulq_f32(vrsqrtx, vx);
150     float32x4_t vhalfrsqrtx = vmulq_f32(vrsqrtx, vhalf);
151     float32x4_t vresidual = vfmsq_f32(vhalf, vsqrtx, vhalfrsqrtx);
152     vhalfrsqrtx = vfmaq_f32(vhalfrsqrtx, vresidual, vhalfrsqrtx);
153     vsqrtx = vfmaq_f32(vsqrtx, vresidual, vsqrtx);
154     vresidual = vfmsq_f32(vhalf, vsqrtx, vhalfrsqrtx);
155     vhalfrsqrtx = vfmaq_f32(vhalfrsqrtx, vresidual, vhalfrsqrtx);
156     vsqrtx = vfmaq_f32(vsqrtx, vresidual, vsqrtx);
157     const float32x4_t vadjustment = vfmsq_f32(vx, vsqrtx, vsqrtx);
158     const float32x4_t vy = vfmaq_f32(vsqrtx, vhalfrsqrtx, vadjustment);
159 
160     float32x2_t vy_lo = vget_low_f32(vy);
161     if (n & (2 * sizeof(float))) {
162       vst1_f32(y, vy_lo); y += 2;
163       vy_lo = vget_high_f32(vy);
164     }
165     if (n & (1 * sizeof(float))) {
166       vst1_lane_f32(y, vy_lo, 0);
167     }
168   }
169 }
170