1*4bdc9457SAndroid Build Coastguard Worker// Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker//   Template: src/qs8-igemm/4x8-aarch32-neon-mlal-lane-ld64.S.in
3*4bdc9457SAndroid Build Coastguard Worker//   Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker//
5*4bdc9457SAndroid Build Coastguard Worker// Copyright 2021 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker//
7*4bdc9457SAndroid Build Coastguard Worker// This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker// LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker
11*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/assembly.h>
12*4bdc9457SAndroid Build Coastguard Worker
13*4bdc9457SAndroid Build Coastguard Worker.syntax unified
14*4bdc9457SAndroid Build Coastguard Worker
15*4bdc9457SAndroid Build Coastguard Worker// void xnn_qc8_igemm_minmax_fp32_ukernel_4x8__aarch32_neon_mlal_lane_prfm_ld64
16*4bdc9457SAndroid Build Coastguard Worker//     size_t mr,                                     (r0)
17*4bdc9457SAndroid Build Coastguard Worker//     size_t nc,                                      r1
18*4bdc9457SAndroid Build Coastguard Worker//     size_t kc,                                     (r2) -> r5 -> sp + 44
19*4bdc9457SAndroid Build Coastguard Worker//     size_t ks,                                     (r3) -> sp + 48 -> r14
20*4bdc9457SAndroid Build Coastguard Worker//     const int8_t**restrict a,            sp + 88  -> r2
21*4bdc9457SAndroid Build Coastguard Worker//     const void*restrict w,              sp + 92  -> r9
22*4bdc9457SAndroid Build Coastguard Worker//     int8_t*restrict c,                   sp + 96  -> r11
23*4bdc9457SAndroid Build Coastguard Worker//     size_t cm_stride,                   sp + 100  -> (r6)
24*4bdc9457SAndroid Build Coastguard Worker//     size_t cn_stride,                   sp + 104  -> (r7)
25*4bdc9457SAndroid Build Coastguard Worker//     size_t a_offset,                    sp + 108 -> (r5)
26*4bdc9457SAndroid Build Coastguard Worker//     const int8_t* zero,                  sp + 112 -> (r7)
27*4bdc9457SAndroid Build Coastguard Worker//     xnn_qs8_minmax_params*params); sp + 116 -> (r5)
28*4bdc9457SAndroid Build Coastguard Worker
29*4bdc9457SAndroid Build Coastguard Worker// d8-d15, r4-r11,r14(lr) need to be preserved if used. r13(sp),r15(pc) are reserved.
30*4bdc9457SAndroid Build Coastguard Worker
31*4bdc9457SAndroid Build Coastguard Worker// Register usage
32*4bdc9457SAndroid Build Coastguard Worker// A0   r3  d0-d1 q0
33*4bdc9457SAndroid Build Coastguard Worker// A1  r12  d2-d3 q1
34*4bdc9457SAndroid Build Coastguard Worker// A2  r10  d4-d5 q2
35*4bdc9457SAndroid Build Coastguard Worker// A3   r0  d6-d7 q3
36*4bdc9457SAndroid Build Coastguard Worker
37*4bdc9457SAndroid Build Coastguard Worker// B    r9  d10-d11 q5
38*4bdc9457SAndroid Build Coastguard Worker
39*4bdc9457SAndroid Build Coastguard Worker// C0  r11 d16-d17  q8  d18-d19  q9
40*4bdc9457SAndroid Build Coastguard Worker// C1   r4 d20-d21 q10  d22-d23 q11
41*4bdc9457SAndroid Build Coastguard Worker// C2   r8 d24-d25 q12  d26-d27 q13
42*4bdc9457SAndroid Build Coastguard Worker// C3   r6 d28-d29 q14  d30-d31 q15
43*4bdc9457SAndroid Build Coastguard Worker
44*4bdc9457SAndroid Build Coastguard Worker// Unused d13-d15
45*4bdc9457SAndroid Build Coastguard Worker
46*4bdc9457SAndroid Build Coastguard Worker// params structure is 10 bytes
47*4bdc9457SAndroid Build Coastguard Worker// struct {
48*4bdc9457SAndroid Build Coastguard Worker//   float magic_bias;                           d12[0]
49*4bdc9457SAndroid Build Coastguard Worker//   int32_t magic_bias_less_output_zero_point;  d12[1]
50*4bdc9457SAndroid Build Coastguard Worker//   int8_t output_min;                          d13[6]
51*4bdc9457SAndroid Build Coastguard Worker//   int8_t output_max;                          d13[7]
52*4bdc9457SAndroid Build Coastguard Worker// } xnn_qs8_minmax_params.neon;
53*4bdc9457SAndroid Build Coastguard Worker
54*4bdc9457SAndroid Build Coastguard WorkerBEGIN_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_4x8__aarch32_neon_mlal_lane_prfm_ld64
55*4bdc9457SAndroid Build Coastguard Worker        # Push 88 bytes
56*4bdc9457SAndroid Build Coastguard Worker        # r2 will be reloaded in outer loop.  r3 is ks
57*4bdc9457SAndroid Build Coastguard Worker        PUSH    {r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, lr}   // +44
58*4bdc9457SAndroid Build Coastguard Worker        SUB     sp, sp, 12                          // +12
59*4bdc9457SAndroid Build Coastguard Worker        VPUSH   {d10-d13}                           // +32 = 88
60*4bdc9457SAndroid Build Coastguard Worker
61*4bdc9457SAndroid Build Coastguard Worker        LDR     r11, [sp, 96]           // c
62*4bdc9457SAndroid Build Coastguard Worker        LDR     r6, [sp, 100]           // cm_stride
63*4bdc9457SAndroid Build Coastguard Worker        LDR     r2, [sp, 88]            // a
64*4bdc9457SAndroid Build Coastguard Worker        LDR     r9, [sp, 92]            // w
65*4bdc9457SAndroid Build Coastguard Worker        LDR     r5, [sp, 116]           // params
66*4bdc9457SAndroid Build Coastguard Worker        MOV     r14, r3                 // p = ks
67*4bdc9457SAndroid Build Coastguard Worker
68*4bdc9457SAndroid Build Coastguard Worker        # Clamp C pointers
69*4bdc9457SAndroid Build Coastguard Worker        CMP     r0, 2                   // if mr >= 2
70*4bdc9457SAndroid Build Coastguard Worker        ADD     r4, r11, r6             //   c1 = c0 + cm_stride
71*4bdc9457SAndroid Build Coastguard Worker        MOVLO   r4, r11                 // c1
72*4bdc9457SAndroid Build Coastguard Worker                                        // if mr > 2
73*4bdc9457SAndroid Build Coastguard Worker        ADD     r8, r4, r6              //   c2 = c1 + cm_stride
74*4bdc9457SAndroid Build Coastguard Worker        MOVLS   r8, r4                  // c2
75*4bdc9457SAndroid Build Coastguard Worker        CMP     r0, 4                   // if mr >=4
76*4bdc9457SAndroid Build Coastguard Worker        ADD     r6, r8, r6              //   c3 = c2 + cm_stride
77*4bdc9457SAndroid Build Coastguard Worker        MOVLO   r6, r8                  // c3
78*4bdc9457SAndroid Build Coastguard Worker
79*4bdc9457SAndroid Build Coastguard Worker        # Load params values
80*4bdc9457SAndroid Build Coastguard Worker        VLDM    r5!, {d12}              // QC8 neon params
81*4bdc9457SAndroid Build Coastguard Worker        VLD1.16 {d13[]}, [r5]
82*4bdc9457SAndroid Build Coastguard Worker
83*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9,  64]               // Prefetch B
84*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9, 128]
85*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9, 192]
86*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9, 256]
87*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9, 320]
88*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9, 384]
89*4bdc9457SAndroid Build Coastguard Worker
90*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
91*4bdc9457SAndroid Build Coastguard Worker0:
92*4bdc9457SAndroid Build Coastguard Worker        # Load initial bias from w into accumulators
93*4bdc9457SAndroid Build Coastguard Worker        VLDM    r9!, {d16-d19}          // Bias
94*4bdc9457SAndroid Build Coastguard Worker        VMOV    q10, q8
95*4bdc9457SAndroid Build Coastguard Worker        VMOV    q11, q9
96*4bdc9457SAndroid Build Coastguard Worker        VMOV    q12, q8
97*4bdc9457SAndroid Build Coastguard Worker        VMOV    q13, q9
98*4bdc9457SAndroid Build Coastguard Worker        VMOV    q14, q8
99*4bdc9457SAndroid Build Coastguard Worker        VMOV    q15, q9
100*4bdc9457SAndroid Build Coastguard Worker
101*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
102*4bdc9457SAndroid Build Coastguard Worker1:
103*4bdc9457SAndroid Build Coastguard Worker        # Load next 4 A pointers
104*4bdc9457SAndroid Build Coastguard Worker        LDR     r3, [r2,  0]
105*4bdc9457SAndroid Build Coastguard Worker        LDR     r12, [r2,  4]
106*4bdc9457SAndroid Build Coastguard Worker        LDR     r10, [r2,  8]
107*4bdc9457SAndroid Build Coastguard Worker        LDR     r0, [r2, 12]
108*4bdc9457SAndroid Build Coastguard Worker        ADD     r2, r2, 16
109*4bdc9457SAndroid Build Coastguard Worker
110*4bdc9457SAndroid Build Coastguard Worker        PLD     [r3, 64]
111*4bdc9457SAndroid Build Coastguard Worker        PLD     [r12, 64]
112*4bdc9457SAndroid Build Coastguard Worker        PLD     [r10, 64]
113*4bdc9457SAndroid Build Coastguard Worker        PLD     [r0, 64]
114*4bdc9457SAndroid Build Coastguard Worker
115*4bdc9457SAndroid Build Coastguard Worker        # Add a_offset
116*4bdc9457SAndroid Build Coastguard Worker        LDR     r5, [sp, 108]           // a_offset
117*4bdc9457SAndroid Build Coastguard Worker        LDR     r7, [sp, 112]           // zero
118*4bdc9457SAndroid Build Coastguard Worker        CMP     r3,  r7                 // if a0 == zero
119*4bdc9457SAndroid Build Coastguard Worker        ADD     r3,  r3, r5             // a0 += a_offset
120*4bdc9457SAndroid Build Coastguard Worker        MOVEQ   r3,  r7                 //   a0 = zero, else += a0 + a_offset
121*4bdc9457SAndroid Build Coastguard Worker        CMP     r12,  r7                // if a1 == zero
122*4bdc9457SAndroid Build Coastguard Worker        ADD     r12, r12, r5            // a1 += a_offset
123*4bdc9457SAndroid Build Coastguard Worker        MOVEQ   r12,  r7                //   a1 = zero, else += a1 + a_offset
124*4bdc9457SAndroid Build Coastguard Worker        CMP     r10,  r7                // if a2 == zero
125*4bdc9457SAndroid Build Coastguard Worker        ADD     r10, r10, r5            // a2 += a_offset
126*4bdc9457SAndroid Build Coastguard Worker        MOVEQ   r10,  r7                //   a2 = zero, else += a2 + a_offset
127*4bdc9457SAndroid Build Coastguard Worker        CMP     r0,  r7                 // if a3 == zero
128*4bdc9457SAndroid Build Coastguard Worker        ADD     r0,  r0, r5             // a3 += a_offset
129*4bdc9457SAndroid Build Coastguard Worker        LDR     r5, [sp, 44]            // kc
130*4bdc9457SAndroid Build Coastguard Worker        MOVEQ   r0,  r7                 //   a3 = zero, else += a3 + a_offset
131*4bdc9457SAndroid Build Coastguard Worker
132*4bdc9457SAndroid Build Coastguard Worker        SUBS    r5, r5, 8               // kc - 8
133*4bdc9457SAndroid Build Coastguard Worker        BLO     4f                      // less than 8 channels?
134*4bdc9457SAndroid Build Coastguard Worker
135*4bdc9457SAndroid Build Coastguard Worker        # Main loop - 8 bytes
136*4bdc9457SAndroid Build Coastguard Worker        # 64 bytes for weights.
137*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
138*4bdc9457SAndroid Build Coastguard Worker2:
139*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d0},  [r3]!            // A0
140*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!           // B
141*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d2}, [r12]!            // A1
142*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d4}, [r10]!            // A2
143*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d6},  [r0]!            // A3
144*4bdc9457SAndroid Build Coastguard Worker        SUBS    r5, r5, 8
145*4bdc9457SAndroid Build Coastguard Worker        PLD     [r3, 128]
146*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q0, d0
147*4bdc9457SAndroid Build Coastguard Worker        PLD     [r12, 128]
148*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
149*4bdc9457SAndroid Build Coastguard Worker        PLD     [r10, 128]
150*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q1, d2
151*4bdc9457SAndroid Build Coastguard Worker        PLD     [r0, 128]
152*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q2, d4
153*4bdc9457SAndroid Build Coastguard Worker        PLD     [r9, 448]
154*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q3, d6
155*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[0]
156*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[0]
157*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[0]
158*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[0]
159*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[0]
160*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[0]
161*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[0]
162*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[0]
163*4bdc9457SAndroid Build Coastguard Worker
164*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
165*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
166*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[1]
167*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[1]
168*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[1]
169*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[1]
170*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[1]
171*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[1]
172*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[1]
173*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[1]
174*4bdc9457SAndroid Build Coastguard Worker
175*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
176*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
177*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[2]
178*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[2]
179*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[2]
180*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[2]
181*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[2]
182*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[2]
183*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[2]
184*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[2]
185*4bdc9457SAndroid Build Coastguard Worker
186*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
187*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
188*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[3]
189*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[3]
190*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[3]
191*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[3]
192*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[3]
193*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[3]
194*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[3]
195*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[3]
196*4bdc9457SAndroid Build Coastguard Worker
197*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
198*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
199*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[0]
200*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[0]
201*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[0]
202*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[0]
203*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[0]
204*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[0]
205*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[0]
206*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[0]
207*4bdc9457SAndroid Build Coastguard Worker
208*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
209*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
210*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[1]
211*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[1]
212*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[1]
213*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[1]
214*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[1]
215*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[1]
216*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[1]
217*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[1]
218*4bdc9457SAndroid Build Coastguard Worker
219*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
220*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
221*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[2]
222*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[2]
223*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[2]
224*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[2]
225*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[2]
226*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[2]
227*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[2]
228*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[2]
229*4bdc9457SAndroid Build Coastguard Worker
230*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
231*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
232*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[3]
233*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[3]
234*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[3]
235*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[3]
236*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[3]
237*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[3]
238*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[3]
239*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[3]
240*4bdc9457SAndroid Build Coastguard Worker        BHS     2b
241*4bdc9457SAndroid Build Coastguard Worker
242*4bdc9457SAndroid Build Coastguard Worker        # Is there a remainder?- 1-7 bytes of A
243*4bdc9457SAndroid Build Coastguard Worker        ADDS    r5, r5, 8
244*4bdc9457SAndroid Build Coastguard Worker        BNE     4f
245*4bdc9457SAndroid Build Coastguard Worker
246*4bdc9457SAndroid Build Coastguard Worker3:
247*4bdc9457SAndroid Build Coastguard Worker        # ks loop
248*4bdc9457SAndroid Build Coastguard Worker        SUBS    r14, r14, 16            // ks -= MR * sizeof(void*)
249*4bdc9457SAndroid Build Coastguard Worker        BHI     1b
250*4bdc9457SAndroid Build Coastguard Worker
251*4bdc9457SAndroid Build Coastguard Worker        LDR     r7, [sp, 104]           // cn_stride
252*4bdc9457SAndroid Build Coastguard Worker        LDR     r14, [sp, 48]           // p = ks
253*4bdc9457SAndroid Build Coastguard Worker
254*4bdc9457SAndroid Build Coastguard Worker        # QC8 FP32 quantization
255*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {q0-q1},  [r9]!
256*4bdc9457SAndroid Build Coastguard Worker
257*4bdc9457SAndroid Build Coastguard Worker        VDUP.32 q2, d12[0]              // magic_bias
258*4bdc9457SAndroid Build Coastguard Worker        VDUP.32 q3, d12[1]              // magic_bias_less_output_zero_point
259*4bdc9457SAndroid Build Coastguard Worker
260*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q8,  q8
261*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q9,  q9
262*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q10, q10
263*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q11, q11
264*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q12, q12
265*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q13, q13
266*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q14, q14
267*4bdc9457SAndroid Build Coastguard Worker        VCVT.F32.S32 q15, q15
268*4bdc9457SAndroid Build Coastguard Worker
269*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q8,  q8, q0            // multiplier
270*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q9,  q9, q1
271*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q10, q10, q0
272*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q11, q11, q1
273*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q12, q12, q0
274*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q13, q13, q1
275*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q14, q14, q0
276*4bdc9457SAndroid Build Coastguard Worker        VMUL.F32 q15, q15, q1
277*4bdc9457SAndroid Build Coastguard Worker
278*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q8,  q8, q2            // magic_bias
279*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q9,  q9, q2
280*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q10, q10, q2
281*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q11, q11, q2
282*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q12, q12, q2
283*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q13, q13, q2
284*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q14, q14, q2
285*4bdc9457SAndroid Build Coastguard Worker        VADD.F32 q15, q15, q2
286*4bdc9457SAndroid Build Coastguard Worker
287*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q8,  q8, q3           // magic_bias_less_output_zero_point
288*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q9,  q9, q3
289*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q10, q10, q3
290*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q11, q11, q3
291*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q12, q12, q3
292*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q13, q13, q3
293*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q14, q14, q3
294*4bdc9457SAndroid Build Coastguard Worker        VQSUB.S32 q15, q15, q3
295*4bdc9457SAndroid Build Coastguard Worker
296*4bdc9457SAndroid Build Coastguard Worker
297*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d16, q8
298*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d17, q9
299*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d18, q10
300*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d19, q11
301*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d20, q12
302*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d21, q13
303*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d22, q14
304*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S32 d23, q15
305*4bdc9457SAndroid Build Coastguard Worker
306*4bdc9457SAndroid Build Coastguard Worker
307*4bdc9457SAndroid Build Coastguard Worker        VDUP.8  q12, d13[6]             // output_min
308*4bdc9457SAndroid Build Coastguard Worker
309*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S16 d0,  q8
310*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S16 d1,  q9
311*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S16 d2, q10
312*4bdc9457SAndroid Build Coastguard Worker        VQMOVN.S16 d3, q11
313*4bdc9457SAndroid Build Coastguard Worker
314*4bdc9457SAndroid Build Coastguard Worker        VDUP.8  q13, d13[7]             // output_max
315*4bdc9457SAndroid Build Coastguard Worker
316*4bdc9457SAndroid Build Coastguard Worker        VMAX.S8 q0, q0, q12
317*4bdc9457SAndroid Build Coastguard Worker        VMAX.S8 q1, q1, q12
318*4bdc9457SAndroid Build Coastguard Worker
319*4bdc9457SAndroid Build Coastguard Worker        SUBS    r1, r1, 8               // nc -= 8
320*4bdc9457SAndroid Build Coastguard Worker
321*4bdc9457SAndroid Build Coastguard Worker        VMIN.S8 q0, q0, q13
322*4bdc9457SAndroid Build Coastguard Worker        VMIN.S8 q1, q1, q13
323*4bdc9457SAndroid Build Coastguard Worker
324*4bdc9457SAndroid Build Coastguard Worker        # Store full 4 x 8
325*4bdc9457SAndroid Build Coastguard Worker        BLO     5f
326*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d3}, [r6], r7
327*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d2}, [r8], r7
328*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d1}, [r4], r7
329*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d0}, [r11], r7
330*4bdc9457SAndroid Build Coastguard Worker        SUB     r2, r2, r14             // a -= ks
331*4bdc9457SAndroid Build Coastguard Worker        BHI     0b
332*4bdc9457SAndroid Build Coastguard Worker
333*4bdc9457SAndroid Build Coastguard Worker        VPOP    {d10-d13}
334*4bdc9457SAndroid Build Coastguard Worker        ADD     sp, sp, 20                          // skip pad of 12, r2, r3
335*4bdc9457SAndroid Build Coastguard Worker        POP     {r4, r5, r6, r7, r8, r9, r10, r11, pc}
336*4bdc9457SAndroid Build Coastguard Worker
337*4bdc9457SAndroid Build Coastguard Worker        # Remainder- 1 to 7 bytes of A
338*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
339*4bdc9457SAndroid Build Coastguard Worker4:
340*4bdc9457SAndroid Build Coastguard Worker        AND     r5, r5, 7               // kc remainder 1 to 7
341*4bdc9457SAndroid Build Coastguard Worker
342*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d0},  [r3]
343*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
344*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d2}, [r12]
345*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d4}, [r10]
346*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d6},  [r0]
347*4bdc9457SAndroid Build Coastguard Worker
348*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q0, d0
349*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
350*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q1, d2
351*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q2, d4
352*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q3, d6
353*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[0]
354*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[0]
355*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[0]
356*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[0]
357*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[0]
358*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[0]
359*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[0]
360*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[0]
361*4bdc9457SAndroid Build Coastguard Worker        CMP     r5, 2
362*4bdc9457SAndroid Build Coastguard Worker        BLO     3b
363*4bdc9457SAndroid Build Coastguard Worker
364*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
365*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
366*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[1]
367*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[1]
368*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[1]
369*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[1]
370*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[1]
371*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[1]
372*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[1]
373*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[1]
374*4bdc9457SAndroid Build Coastguard Worker        BEQ     3b
375*4bdc9457SAndroid Build Coastguard Worker
376*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
377*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
378*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[2]
379*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[2]
380*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[2]
381*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[2]
382*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[2]
383*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[2]
384*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[2]
385*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[2]
386*4bdc9457SAndroid Build Coastguard Worker        CMP     r5, 4
387*4bdc9457SAndroid Build Coastguard Worker        BLO     3b
388*4bdc9457SAndroid Build Coastguard Worker
389*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
390*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
391*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d0[3]
392*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d0[3]
393*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d2[3]
394*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d2[3]
395*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d4[3]
396*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d4[3]
397*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d6[3]
398*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d6[3]
399*4bdc9457SAndroid Build Coastguard Worker        BEQ     3b
400*4bdc9457SAndroid Build Coastguard Worker
401*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
402*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
403*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[0]
404*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[0]
405*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[0]
406*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[0]
407*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[0]
408*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[0]
409*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[0]
410*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[0]
411*4bdc9457SAndroid Build Coastguard Worker        CMP     r5, 6
412*4bdc9457SAndroid Build Coastguard Worker        BLO     3b
413*4bdc9457SAndroid Build Coastguard Worker
414*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
415*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
416*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[1]
417*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[1]
418*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[1]
419*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[1]
420*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[1]
421*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[1]
422*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[1]
423*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[1]
424*4bdc9457SAndroid Build Coastguard Worker        BEQ     3b
425*4bdc9457SAndroid Build Coastguard Worker
426*4bdc9457SAndroid Build Coastguard Worker        VLD1.8  {d10},  [r9]!
427*4bdc9457SAndroid Build Coastguard Worker        VMOVL.S8 q5, d10
428*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q8, d10, d1[2]
429*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q9, d11, d1[2]
430*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q10, d10, d3[2]
431*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q11, d11, d3[2]
432*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q12, d10, d5[2]
433*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q13, d11, d5[2]
434*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q14, d10, d7[2]
435*4bdc9457SAndroid Build Coastguard Worker        VMLAL.S16 q15, d11, d7[2]
436*4bdc9457SAndroid Build Coastguard Worker        B       3b
437*4bdc9457SAndroid Build Coastguard Worker
438*4bdc9457SAndroid Build Coastguard Worker        # Store odd width
439*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
440*4bdc9457SAndroid Build Coastguard Worker5:
441*4bdc9457SAndroid Build Coastguard Worker        TST     r1, 4
442*4bdc9457SAndroid Build Coastguard Worker        BEQ     6f
443*4bdc9457SAndroid Build Coastguard Worker        VST1.32 {d3[0]}, [r6]!
444*4bdc9457SAndroid Build Coastguard Worker        VST1.32 {d2[0]}, [r8]!
445*4bdc9457SAndroid Build Coastguard Worker        VST1.32 {d1[0]}, [r4]!
446*4bdc9457SAndroid Build Coastguard Worker        VST1.32 {d0[0]}, [r11]!
447*4bdc9457SAndroid Build Coastguard Worker        VEXT.8  q1, q1, q1, 4
448*4bdc9457SAndroid Build Coastguard Worker        VEXT.8  q0, q0, q0, 4
449*4bdc9457SAndroid Build Coastguard Worker6:
450*4bdc9457SAndroid Build Coastguard Worker        TST     r1, 2
451*4bdc9457SAndroid Build Coastguard Worker        BEQ     7f
452*4bdc9457SAndroid Build Coastguard Worker        VST1.16 {d3[0]}, [r6]!
453*4bdc9457SAndroid Build Coastguard Worker        VST1.16 {d2[0]}, [r8]!
454*4bdc9457SAndroid Build Coastguard Worker        VST1.16 {d1[0]}, [r4]!
455*4bdc9457SAndroid Build Coastguard Worker        VST1.16 {d0[0]}, [r11]!
456*4bdc9457SAndroid Build Coastguard Worker        VEXT.8  q1, q1, q1, 2
457*4bdc9457SAndroid Build Coastguard Worker        VEXT.8  q0, q0, q0, 2
458*4bdc9457SAndroid Build Coastguard Worker
459*4bdc9457SAndroid Build Coastguard Worker7:
460*4bdc9457SAndroid Build Coastguard Worker        TST     r1, 1
461*4bdc9457SAndroid Build Coastguard Worker        BEQ     8f
462*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d3[0]}, [r6]
463*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d2[0]}, [r8]
464*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d1[0]}, [r4]
465*4bdc9457SAndroid Build Coastguard Worker        VST1.8  {d0[0]}, [r11]
466*4bdc9457SAndroid Build Coastguard Worker
467*4bdc9457SAndroid Build Coastguard Worker8:
468*4bdc9457SAndroid Build Coastguard Worker        VPOP    {d10-d13}
469*4bdc9457SAndroid Build Coastguard Worker        ADD     sp, sp, 20                          // skip pad of 12, r2, r3
470*4bdc9457SAndroid Build Coastguard Worker        POP     {r4, r5, r6, r7, r8, r9, r10, r11, pc}
471*4bdc9457SAndroid Build Coastguard Worker
472*4bdc9457SAndroid Build Coastguard WorkerEND_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_4x8__aarch32_neon_mlal_lane_prfm_ld64
473*4bdc9457SAndroid Build Coastguard Worker#ifdef __ELF__
474*4bdc9457SAndroid Build Coastguard Worker.section ".note.GNU-stack","",%progbits
475*4bdc9457SAndroid Build Coastguard Worker#endif
476