xref: /aosp_15_r20/external/XNNPACK/src/qc8-igemm/gen/2x8c8-minmax-fp32-aarch64-neon-mlal-prfm.S (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1*4bdc9457SAndroid Build Coastguard Worker// Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker//   Template: src/qs8-igemm/2x8c8-aarch64-neon-mlal.S.in
3*4bdc9457SAndroid Build Coastguard Worker//   Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker//
5*4bdc9457SAndroid Build Coastguard Worker// Copyright 2021 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker//
7*4bdc9457SAndroid Build Coastguard Worker// This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker// LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker
11*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/assembly.h>
12*4bdc9457SAndroid Build Coastguard Worker
13*4bdc9457SAndroid Build Coastguard Worker# void xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal(
14*4bdc9457SAndroid Build Coastguard Worker#     size_t mr,                 x0
15*4bdc9457SAndroid Build Coastguard Worker#     size_t nc,                 x1
16*4bdc9457SAndroid Build Coastguard Worker#     size_t kc,                 x2 / x0
17*4bdc9457SAndroid Build Coastguard Worker#     size_t ks,                 x3 / x9
18*4bdc9457SAndroid Build Coastguard Worker#     const int8_t**restrict a,  x4
19*4bdc9457SAndroid Build Coastguard Worker#     const int8_t* restrict w,  x5
20*4bdc9457SAndroid Build Coastguard Worker#     int8_t* restrict c,        x6
21*4bdc9457SAndroid Build Coastguard Worker#     size_t cm_stride,          x7
22*4bdc9457SAndroid Build Coastguard Worker#     size_t cn_stride,                  [sp] -> x10
23*4bdc9457SAndroid Build Coastguard Worker#     size_t a_offset,                   [sp + 8] -> x8
24*4bdc9457SAndroid Build Coastguard Worker#     const int8_t* zero,                [sp + 16] -> x12
25*4bdc9457SAndroid Build Coastguard Worker#     const union xnn_qs8_minmax_params params [sp + 24] -> x11
26*4bdc9457SAndroid Build Coastguard Worker
27*4bdc9457SAndroid Build Coastguard Worker# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28*4bdc9457SAndroid Build Coastguard Worker
29*4bdc9457SAndroid Build Coastguard Worker# Register usage
30*4bdc9457SAndroid Build Coastguard Worker# A0 x13  v0  v6
31*4bdc9457SAndroid Build Coastguard Worker# A1 x15  v1  v7
32*4bdc9457SAndroid Build Coastguard Worker# B   x5  v4  v5  v8  v9
33*4bdc9457SAndroid Build Coastguard Worker# C0  x6 v16 v18 v20 v22 v24 v26 v28 v30
34*4bdc9457SAndroid Build Coastguard Worker# C1  x7 v17 v19 v21 v23 v25 v27 v29 v31
35*4bdc9457SAndroid Build Coastguard Worker# temp0   v2 v10 v12 v14
36*4bdc9457SAndroid Build Coastguard Worker# temp1   v3 v11 v13 v15
37*4bdc9457SAndroid Build Coastguard Worker
38*4bdc9457SAndroid Build Coastguard WorkerBEGIN_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal_prfm
39*4bdc9457SAndroid Build Coastguard Worker
40*4bdc9457SAndroid Build Coastguard Worker        # Clamp C pointers
41*4bdc9457SAndroid Build Coastguard Worker        LDP     x10, x8, [sp]           // Load cn_stride, a_offset
42*4bdc9457SAndroid Build Coastguard Worker        CMP     x0, 2                   // if mr < 2
43*4bdc9457SAndroid Build Coastguard Worker        LDP     x12, x11, [sp, 16]      // Load zero, params pointer
44*4bdc9457SAndroid Build Coastguard Worker        ADD     x7, x6, x7              // c1 = c0 + cm_stride
45*4bdc9457SAndroid Build Coastguard Worker        STP     d8, d9, [sp, -64]!
46*4bdc9457SAndroid Build Coastguard Worker        ADD     x2, x2, 7               // kc = (kc + 7) & ~7
47*4bdc9457SAndroid Build Coastguard Worker        STP     d10, d11, [sp, 16]
48*4bdc9457SAndroid Build Coastguard Worker        CSEL    x7, x6, x7, LO          //   c1 = c0
49*4bdc9457SAndroid Build Coastguard Worker        STP     d12, d13, [sp, 32]
50*4bdc9457SAndroid Build Coastguard Worker        BIC     x2, x2, 7
51*4bdc9457SAndroid Build Coastguard Worker        STP     d14, d15, [sp, 48]
52*4bdc9457SAndroid Build Coastguard Worker
53*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
54*4bdc9457SAndroid Build Coastguard Worker0:
55*4bdc9457SAndroid Build Coastguard Worker        # Load initial bias from w into accumulators
56*4bdc9457SAndroid Build Coastguard Worker        LDP     s16, s18, [x5], 8
57*4bdc9457SAndroid Build Coastguard Worker        MOV     v17.16b, v16.16b
58*4bdc9457SAndroid Build Coastguard Worker        MOV     v19.16b, v18.16b
59*4bdc9457SAndroid Build Coastguard Worker        LDP     s20, s22, [x5], 8
60*4bdc9457SAndroid Build Coastguard Worker        MOV     v21.16b, v20.16b
61*4bdc9457SAndroid Build Coastguard Worker        MOV     v23.16b, v22.16b
62*4bdc9457SAndroid Build Coastguard Worker        LDP     s24, s26, [x5], 8
63*4bdc9457SAndroid Build Coastguard Worker        MOV     v25.16b, v24.16b
64*4bdc9457SAndroid Build Coastguard Worker        MOV     v27.16b, v26.16b
65*4bdc9457SAndroid Build Coastguard Worker        LDP     s28, s30, [x5], 8
66*4bdc9457SAndroid Build Coastguard Worker        MOV     v29.16b, v28.16b
67*4bdc9457SAndroid Build Coastguard Worker        MOV     v31.16b, v30.16b
68*4bdc9457SAndroid Build Coastguard Worker        MOV     x9, x3                  // p = ks
69*4bdc9457SAndroid Build Coastguard Worker
70*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
71*4bdc9457SAndroid Build Coastguard Worker1:
72*4bdc9457SAndroid Build Coastguard Worker        # Load next 2 A pointers
73*4bdc9457SAndroid Build Coastguard Worker        LDP     x13, x15, [x4], 16
74*4bdc9457SAndroid Build Coastguard Worker        CMP     x13, x12                // if a0 == zero
75*4bdc9457SAndroid Build Coastguard Worker        ADD     x13, x13, x8            // a0 += a_offset
76*4bdc9457SAndroid Build Coastguard Worker        CSEL    x13, x12, x13, EQ       //   a0 = zero, else += a0 + a_offset
77*4bdc9457SAndroid Build Coastguard Worker        CMP     x15, x12                // if a1 == zero
78*4bdc9457SAndroid Build Coastguard Worker        ADD     x15, x15, x8            // a1 += a_offset
79*4bdc9457SAndroid Build Coastguard Worker        CSEL    x15, x12, x15, EQ       //   a1 = zero, else += a1 + a_offset
80*4bdc9457SAndroid Build Coastguard Worker
81*4bdc9457SAndroid Build Coastguard Worker        # Is there at least 16 bytes for epilogue?
82*4bdc9457SAndroid Build Coastguard Worker        SUBS    x0, x2, 16              // k = kc - 16
83*4bdc9457SAndroid Build Coastguard Worker        B.LO    5f
84*4bdc9457SAndroid Build Coastguard Worker
85*4bdc9457SAndroid Build Coastguard Worker        # Prologue: load A0, A1 and 2 B's
86*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5]
87*4bdc9457SAndroid Build Coastguard Worker        LDP     d0, d6, [x13], 16
88*4bdc9457SAndroid Build Coastguard Worker        LDP     d1, d7, [x15], 16
89*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 64]
90*4bdc9457SAndroid Build Coastguard Worker
91*4bdc9457SAndroid Build Coastguard Worker        # Is there at least 16 bytes for main loop?
92*4bdc9457SAndroid Build Coastguard Worker        SUBS    x0, x0, 16              // k = k - 16
93*4bdc9457SAndroid Build Coastguard Worker        B.LO    3f
94*4bdc9457SAndroid Build Coastguard Worker
95*4bdc9457SAndroid Build Coastguard Worker         # Main loop - 16 bytes of A
96*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
97*4bdc9457SAndroid Build Coastguard Worker2:
98*4bdc9457SAndroid Build Coastguard Worker        SMULL   v2.8h, v4.8b, v0.8b
99*4bdc9457SAndroid Build Coastguard Worker        SMULL   v3.8h, v4.8b, v1.8b
100*4bdc9457SAndroid Build Coastguard Worker        PRFM    PLDL1KEEP, [x5, 448]
101*4bdc9457SAndroid Build Coastguard Worker        SMULL   v10.8h, v5.8b, v0.8b
102*4bdc9457SAndroid Build Coastguard Worker        SMULL   v11.8h, v5.8b, v1.8b
103*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 16]
104*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v2.8h, v8.8b, v6.8b
105*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v3.8h, v8.8b, v7.8b
106*4bdc9457SAndroid Build Coastguard Worker        PRFM    PLDL1KEEP, [x5, 512]
107*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v10.8h, v9.8b, v6.8b
108*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v11.8h, v9.8b, v7.8b
109*4bdc9457SAndroid Build Coastguard Worker
110*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 80]
111*4bdc9457SAndroid Build Coastguard Worker        SMULL   v12.8h, v4.8b, v0.8b
112*4bdc9457SAndroid Build Coastguard Worker        SADALP  v16.4s,  v2.8h
113*4bdc9457SAndroid Build Coastguard Worker        SMULL   v13.8h, v4.8b, v1.8b
114*4bdc9457SAndroid Build Coastguard Worker        SADALP  v17.4s,  v3.8h
115*4bdc9457SAndroid Build Coastguard Worker        SMULL   v14.8h, v5.8b, v0.8b
116*4bdc9457SAndroid Build Coastguard Worker        SADALP  v18.4s, v10.8h
117*4bdc9457SAndroid Build Coastguard Worker        SMULL   v15.8h, v5.8b, v1.8b
118*4bdc9457SAndroid Build Coastguard Worker        SADALP  v19.4s, v11.8h
119*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 32]
120*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v12.8h, v8.8b, v6.8b
121*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v13.8h, v8.8b, v7.8b
122*4bdc9457SAndroid Build Coastguard Worker        PRFM    PLDL1KEEP, [x13, 128]
123*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v14.8h, v9.8b, v6.8b
124*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v15.8h, v9.8b, v7.8b
125*4bdc9457SAndroid Build Coastguard Worker
126*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 96]
127*4bdc9457SAndroid Build Coastguard Worker        SMULL   v2.8h, v4.8b, v0.8b
128*4bdc9457SAndroid Build Coastguard Worker        SADALP  v20.4s, v12.8h
129*4bdc9457SAndroid Build Coastguard Worker        SMULL   v3.8h, v4.8b, v1.8b
130*4bdc9457SAndroid Build Coastguard Worker        SADALP  v21.4s, v13.8h
131*4bdc9457SAndroid Build Coastguard Worker        SMULL   v10.8h, v5.8b, v0.8b
132*4bdc9457SAndroid Build Coastguard Worker        SADALP  v22.4s, v14.8h
133*4bdc9457SAndroid Build Coastguard Worker        SMULL   v11.8h, v5.8b, v1.8b
134*4bdc9457SAndroid Build Coastguard Worker        SADALP  v23.4s, v15.8h
135*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 48]
136*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v2.8h, v8.8b, v6.8b
137*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v3.8h, v8.8b, v7.8b
138*4bdc9457SAndroid Build Coastguard Worker        PRFM    PLDL1KEEP, [x15, 128]
139*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v10.8h, v9.8b, v6.8b
140*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v11.8h, v9.8b, v7.8b
141*4bdc9457SAndroid Build Coastguard Worker
142*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 112]
143*4bdc9457SAndroid Build Coastguard Worker        SMULL   v12.8h, v4.8b, v0.8b
144*4bdc9457SAndroid Build Coastguard Worker        ADD     x5, x5, 128
145*4bdc9457SAndroid Build Coastguard Worker        SADALP  v24.4s,  v2.8h
146*4bdc9457SAndroid Build Coastguard Worker        SMULL   v13.8h, v4.8b, v1.8b
147*4bdc9457SAndroid Build Coastguard Worker        SADALP  v25.4s,  v3.8h
148*4bdc9457SAndroid Build Coastguard Worker        SMULL   v14.8h, v5.8b, v0.8b
149*4bdc9457SAndroid Build Coastguard Worker        SADALP  v26.4s, v10.8h
150*4bdc9457SAndroid Build Coastguard Worker        SMULL   v15.8h, v5.8b, v1.8b
151*4bdc9457SAndroid Build Coastguard Worker        SADALP  v27.4s, v11.8h
152*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v12.8h, v8.8b, v6.8b
153*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5]            // Read B
154*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v13.8h, v8.8b, v7.8b
155*4bdc9457SAndroid Build Coastguard Worker        SUBS    x0, x0, 16
156*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v14.8h, v9.8b, v6.8b
157*4bdc9457SAndroid Build Coastguard Worker        LDP     d0, d6, [x13], 16       // Read A0
158*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v15.8h, v9.8b, v7.8b
159*4bdc9457SAndroid Build Coastguard Worker
160*4bdc9457SAndroid Build Coastguard Worker        SADALP  v28.4s, v12.8h
161*4bdc9457SAndroid Build Coastguard Worker        LDP     d1, d7, [x15], 16       // Read A1
162*4bdc9457SAndroid Build Coastguard Worker        SADALP  v29.4s, v13.8h
163*4bdc9457SAndroid Build Coastguard Worker        SADALP  v30.4s, v14.8h
164*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 64]        // Read B
165*4bdc9457SAndroid Build Coastguard Worker        SADALP  v31.4s, v15.8h
166*4bdc9457SAndroid Build Coastguard Worker        B.HS    2b
167*4bdc9457SAndroid Build Coastguard Worker
168*4bdc9457SAndroid Build Coastguard Worker        # Epilogue
169*4bdc9457SAndroid Build Coastguard Worker        # Same as main loop except no loads at end of loop
170*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
171*4bdc9457SAndroid Build Coastguard Worker3:
172*4bdc9457SAndroid Build Coastguard Worker        SMULL   v2.8h, v4.8b, v0.8b
173*4bdc9457SAndroid Build Coastguard Worker        SMULL   v3.8h, v4.8b, v1.8b
174*4bdc9457SAndroid Build Coastguard Worker        SMULL   v10.8h, v5.8b, v0.8b
175*4bdc9457SAndroid Build Coastguard Worker        SMULL   v11.8h, v5.8b, v1.8b
176*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 16]
177*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v2.8h, v8.8b, v6.8b
178*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v3.8h, v8.8b, v7.8b
179*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v10.8h, v9.8b, v6.8b
180*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v11.8h, v9.8b, v7.8b
181*4bdc9457SAndroid Build Coastguard Worker
182*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 80]
183*4bdc9457SAndroid Build Coastguard Worker        SMULL   v12.8h, v4.8b, v0.8b
184*4bdc9457SAndroid Build Coastguard Worker        SADALP  v16.4s,  v2.8h
185*4bdc9457SAndroid Build Coastguard Worker        SMULL   v13.8h, v4.8b, v1.8b
186*4bdc9457SAndroid Build Coastguard Worker        SADALP  v17.4s,  v3.8h
187*4bdc9457SAndroid Build Coastguard Worker        SMULL   v14.8h, v5.8b, v0.8b
188*4bdc9457SAndroid Build Coastguard Worker        SADALP  v18.4s, v10.8h
189*4bdc9457SAndroid Build Coastguard Worker        SMULL   v15.8h, v5.8b, v1.8b
190*4bdc9457SAndroid Build Coastguard Worker        SADALP  v19.4s, v11.8h
191*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 32]
192*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v12.8h, v8.8b, v6.8b
193*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v13.8h, v8.8b, v7.8b
194*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v14.8h, v9.8b, v6.8b
195*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v15.8h, v9.8b, v7.8b
196*4bdc9457SAndroid Build Coastguard Worker
197*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 96]
198*4bdc9457SAndroid Build Coastguard Worker        SMULL   v2.8h, v4.8b, v0.8b
199*4bdc9457SAndroid Build Coastguard Worker        SADALP  v20.4s, v12.8h
200*4bdc9457SAndroid Build Coastguard Worker        SMULL   v3.8h, v4.8b, v1.8b
201*4bdc9457SAndroid Build Coastguard Worker        SADALP  v21.4s, v13.8h
202*4bdc9457SAndroid Build Coastguard Worker        SMULL   v10.8h, v5.8b, v0.8b
203*4bdc9457SAndroid Build Coastguard Worker        SADALP  v22.4s, v14.8h
204*4bdc9457SAndroid Build Coastguard Worker        SMULL   v11.8h, v5.8b, v1.8b
205*4bdc9457SAndroid Build Coastguard Worker        SADALP  v23.4s, v15.8h
206*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 48]
207*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v2.8h, v8.8b, v6.8b
208*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v3.8h, v8.8b, v7.8b
209*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v10.8h, v9.8b, v6.8b
210*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v11.8h, v9.8b, v7.8b
211*4bdc9457SAndroid Build Coastguard Worker
212*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [x5, 112]
213*4bdc9457SAndroid Build Coastguard Worker        SMULL   v12.8h, v4.8b, v0.8b
214*4bdc9457SAndroid Build Coastguard Worker        SADALP  v24.4s,  v2.8h
215*4bdc9457SAndroid Build Coastguard Worker        SMULL   v13.8h, v4.8b, v1.8b
216*4bdc9457SAndroid Build Coastguard Worker        SADALP  v25.4s,  v3.8h
217*4bdc9457SAndroid Build Coastguard Worker        SMULL   v14.8h, v5.8b, v0.8b
218*4bdc9457SAndroid Build Coastguard Worker        SADALP  v26.4s, v10.8h
219*4bdc9457SAndroid Build Coastguard Worker        SMULL   v15.8h, v5.8b, v1.8b
220*4bdc9457SAndroid Build Coastguard Worker        SADALP  v27.4s, v11.8h
221*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v12.8h, v8.8b, v6.8b
222*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v13.8h, v8.8b, v7.8b
223*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v14.8h, v9.8b, v6.8b
224*4bdc9457SAndroid Build Coastguard Worker        SMLAL   v15.8h, v9.8b, v7.8b
225*4bdc9457SAndroid Build Coastguard Worker        ADD     x5, x5, 128
226*4bdc9457SAndroid Build Coastguard Worker
227*4bdc9457SAndroid Build Coastguard Worker        SADALP  v28.4s, v12.8h
228*4bdc9457SAndroid Build Coastguard Worker        SADALP  v29.4s, v13.8h
229*4bdc9457SAndroid Build Coastguard Worker        SADALP  v30.4s, v14.8h
230*4bdc9457SAndroid Build Coastguard Worker        SADALP  v31.4s, v15.8h
231*4bdc9457SAndroid Build Coastguard Worker
232*4bdc9457SAndroid Build Coastguard Worker        # Is there a remainder?- 8 bytes of A
233*4bdc9457SAndroid Build Coastguard Worker        TBNZ    x0, 3, 5f
234*4bdc9457SAndroid Build Coastguard Worker
235*4bdc9457SAndroid Build Coastguard Worker        # ks loop
236*4bdc9457SAndroid Build Coastguard Worker        SUBS    x9, x9, 16              // ks -= MR * sizeof(int8_t*)
237*4bdc9457SAndroid Build Coastguard Worker        B.HI    1b
238*4bdc9457SAndroid Build Coastguard Worker
239*4bdc9457SAndroid Build Coastguard Worker4:
240*4bdc9457SAndroid Build Coastguard Worker        # Add columns
241*4bdc9457SAndroid Build Coastguard Worker        ADDP    v16.4s, v16.4s, v18.4s
242*4bdc9457SAndroid Build Coastguard Worker        ADDP    v20.4s, v20.4s, v22.4s
243*4bdc9457SAndroid Build Coastguard Worker        ADDP    v24.4s, v24.4s, v26.4s
244*4bdc9457SAndroid Build Coastguard Worker        ADDP    v28.4s, v28.4s, v30.4s
245*4bdc9457SAndroid Build Coastguard Worker        ADDP    v17.4s, v17.4s, v19.4s
246*4bdc9457SAndroid Build Coastguard Worker        ADDP    v21.4s, v21.4s, v23.4s
247*4bdc9457SAndroid Build Coastguard Worker        ADDP    v25.4s, v25.4s, v27.4s
248*4bdc9457SAndroid Build Coastguard Worker        ADDP    v29.4s, v29.4s, v31.4s
249*4bdc9457SAndroid Build Coastguard Worker        ADDP    v0.4s, v16.4s, v20.4s
250*4bdc9457SAndroid Build Coastguard Worker        ADDP    v1.4s, v24.4s, v28.4s
251*4bdc9457SAndroid Build Coastguard Worker        ADDP    v2.4s, v17.4s, v21.4s
252*4bdc9457SAndroid Build Coastguard Worker        ADDP    v3.4s, v25.4s, v29.4s
253*4bdc9457SAndroid Build Coastguard Worker
254*4bdc9457SAndroid Build Coastguard Worker        # Load per channel scale values from weights
255*4bdc9457SAndroid Build Coastguard Worker        SCVTF   v0.4s, v0.4s
256*4bdc9457SAndroid Build Coastguard Worker        LDR     q4, [x5], 16
257*4bdc9457SAndroid Build Coastguard Worker        SCVTF   v1.4s, v1.4s
258*4bdc9457SAndroid Build Coastguard Worker        LDR     q5, [x5], 16
259*4bdc9457SAndroid Build Coastguard Worker        SCVTF   v2.4s, v2.4s
260*4bdc9457SAndroid Build Coastguard Worker        SCVTF   v3.4s, v3.4s
261*4bdc9457SAndroid Build Coastguard Worker        FMUL    v0.4s, v0.4s, v4.4s
262*4bdc9457SAndroid Build Coastguard Worker        FMUL    v1.4s, v1.4s, v5.4s
263*4bdc9457SAndroid Build Coastguard Worker        FMUL    v2.4s, v2.4s, v4.4s
264*4bdc9457SAndroid Build Coastguard Worker        FMUL    v3.4s, v3.4s, v5.4s
265*4bdc9457SAndroid Build Coastguard Worker
266*4bdc9457SAndroid Build Coastguard Worker        FCVTNS  v0.4s, v0.4s
267*4bdc9457SAndroid Build Coastguard Worker        FCVTNS  v1.4s, v1.4s
268*4bdc9457SAndroid Build Coastguard Worker        FCVTNS  v2.4s, v2.4s
269*4bdc9457SAndroid Build Coastguard Worker        FCVTNS  v3.4s, v3.4s
270*4bdc9457SAndroid Build Coastguard Worker
271*4bdc9457SAndroid Build Coastguard Worker        LD1R    {v5.8h}, [x11], 2
272*4bdc9457SAndroid Build Coastguard Worker        SQXTN   v0.4h, v0.4s
273*4bdc9457SAndroid Build Coastguard Worker        SQXTN   v2.4h, v2.4s
274*4bdc9457SAndroid Build Coastguard Worker        SQXTN2  v0.8h, v1.4s
275*4bdc9457SAndroid Build Coastguard Worker        SQXTN2  v2.8h, v3.4s
276*4bdc9457SAndroid Build Coastguard Worker        SUBS    x1, x1, 8
277*4bdc9457SAndroid Build Coastguard Worker        SQADD   v0.8h, v0.8h, v5.8h
278*4bdc9457SAndroid Build Coastguard Worker        SQADD   v1.8h, v2.8h, v5.8h
279*4bdc9457SAndroid Build Coastguard Worker        SQXTN   v0.8b, v0.8h
280*4bdc9457SAndroid Build Coastguard Worker        SQXTN2  v0.16b, v1.8h
281*4bdc9457SAndroid Build Coastguard Worker        LD1R    {v1.16b}, [x11], 1
282*4bdc9457SAndroid Build Coastguard Worker        LD1R    {v2.16b}, [x11]
283*4bdc9457SAndroid Build Coastguard Worker        SMAX    v0.16b, v0.16b, v1.16b
284*4bdc9457SAndroid Build Coastguard Worker        SUB     x11, x11, 3          // rewind params pointer
285*4bdc9457SAndroid Build Coastguard Worker        SMIN    v0.16b, v0.16b, v2.16b
286*4bdc9457SAndroid Build Coastguard Worker        B.LO    6f
287*4bdc9457SAndroid Build Coastguard Worker
288*4bdc9457SAndroid Build Coastguard Worker        # Store full 2 x 8
289*4bdc9457SAndroid Build Coastguard Worker        ST1     {v0.d}[1], [x7], x10
290*4bdc9457SAndroid Build Coastguard Worker        ST1     {v0.8b}, [x6], x10
291*4bdc9457SAndroid Build Coastguard Worker
292*4bdc9457SAndroid Build Coastguard Worker        SUB     x4, x4, x3              // a -= ks
293*4bdc9457SAndroid Build Coastguard Worker
294*4bdc9457SAndroid Build Coastguard Worker        # nc loop
295*4bdc9457SAndroid Build Coastguard Worker        B.HI    0b
296*4bdc9457SAndroid Build Coastguard Worker
297*4bdc9457SAndroid Build Coastguard Worker        # Restore d8-d15 from stack
298*4bdc9457SAndroid Build Coastguard Worker        LDP     d14, d15, [sp, 48]
299*4bdc9457SAndroid Build Coastguard Worker        LDP     d12, d13, [sp, 32]
300*4bdc9457SAndroid Build Coastguard Worker        LDP     d10, d11, [sp, 16]
301*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [sp], 64
302*4bdc9457SAndroid Build Coastguard Worker        RET
303*4bdc9457SAndroid Build Coastguard Worker
304*4bdc9457SAndroid Build Coastguard Worker        # Remainder - 8 bytes of A
305*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
306*4bdc9457SAndroid Build Coastguard Worker5:
307*4bdc9457SAndroid Build Coastguard Worker        LDR     d0, [x13]
308*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5]
309*4bdc9457SAndroid Build Coastguard Worker        LDR     d1, [x15]
310*4bdc9457SAndroid Build Coastguard Worker        LDP     d6, d7, [x5, 16]
311*4bdc9457SAndroid Build Coastguard Worker        SMULL   v2.8h, v4.8b, v0.8b
312*4bdc9457SAndroid Build Coastguard Worker        SMULL   v3.8h, v4.8b, v1.8b
313*4bdc9457SAndroid Build Coastguard Worker        SMULL   v10.8h, v5.8b, v0.8b
314*4bdc9457SAndroid Build Coastguard Worker        SMULL   v11.8h, v5.8b, v1.8b
315*4bdc9457SAndroid Build Coastguard Worker        SMULL   v12.8h, v6.8b, v0.8b
316*4bdc9457SAndroid Build Coastguard Worker        SADALP  v16.4s,  v2.8h
317*4bdc9457SAndroid Build Coastguard Worker        SMULL   v13.8h, v6.8b, v1.8b
318*4bdc9457SAndroid Build Coastguard Worker        SADALP  v17.4s,  v3.8h
319*4bdc9457SAndroid Build Coastguard Worker        SMULL   v14.8h, v7.8b, v0.8b
320*4bdc9457SAndroid Build Coastguard Worker        SADALP  v18.4s, v10.8h
321*4bdc9457SAndroid Build Coastguard Worker        SMULL   v15.8h, v7.8b, v1.8b
322*4bdc9457SAndroid Build Coastguard Worker        SADALP  v19.4s, v11.8h
323*4bdc9457SAndroid Build Coastguard Worker        LDP     d4, d5, [x5, 32]
324*4bdc9457SAndroid Build Coastguard Worker        SMULL   v2.8h, v4.8b, v0.8b
325*4bdc9457SAndroid Build Coastguard Worker        SADALP  v20.4s, v12.8h
326*4bdc9457SAndroid Build Coastguard Worker        SMULL   v3.8h, v4.8b, v1.8b
327*4bdc9457SAndroid Build Coastguard Worker        SADALP  v21.4s, v13.8h
328*4bdc9457SAndroid Build Coastguard Worker        SMULL   v10.8h, v5.8b, v0.8b
329*4bdc9457SAndroid Build Coastguard Worker        SADALP  v22.4s, v14.8h
330*4bdc9457SAndroid Build Coastguard Worker        SMULL   v11.8h, v5.8b, v1.8b
331*4bdc9457SAndroid Build Coastguard Worker        SADALP  v23.4s, v15.8h
332*4bdc9457SAndroid Build Coastguard Worker        LDP     d6, d7, [x5, 48]
333*4bdc9457SAndroid Build Coastguard Worker        SMULL   v12.8h, v6.8b, v0.8b
334*4bdc9457SAndroid Build Coastguard Worker        SADALP  v24.4s,  v2.8h
335*4bdc9457SAndroid Build Coastguard Worker        SMULL   v13.8h, v6.8b, v1.8b
336*4bdc9457SAndroid Build Coastguard Worker        SADALP  v25.4s,  v3.8h
337*4bdc9457SAndroid Build Coastguard Worker        SMULL   v14.8h, v7.8b, v0.8b
338*4bdc9457SAndroid Build Coastguard Worker        SADALP  v26.4s, v10.8h
339*4bdc9457SAndroid Build Coastguard Worker        SMULL   v15.8h, v7.8b, v1.8b
340*4bdc9457SAndroid Build Coastguard Worker        SADALP  v27.4s, v11.8h
341*4bdc9457SAndroid Build Coastguard Worker        ADD     x5, x5, 64
342*4bdc9457SAndroid Build Coastguard Worker        SADALP  v28.4s, v12.8h
343*4bdc9457SAndroid Build Coastguard Worker        SADALP  v29.4s, v13.8h
344*4bdc9457SAndroid Build Coastguard Worker        SADALP  v30.4s, v14.8h
345*4bdc9457SAndroid Build Coastguard Worker        SADALP  v31.4s, v15.8h
346*4bdc9457SAndroid Build Coastguard Worker
347*4bdc9457SAndroid Build Coastguard Worker        # ks loop
348*4bdc9457SAndroid Build Coastguard Worker        SUBS    x9, x9, 16              // ks -= MR * sizeof(int8_t*)
349*4bdc9457SAndroid Build Coastguard Worker        B.HI    1b
350*4bdc9457SAndroid Build Coastguard Worker        B       4b
351*4bdc9457SAndroid Build Coastguard Worker
352*4bdc9457SAndroid Build Coastguard Worker        # Store odd width
353*4bdc9457SAndroid Build Coastguard Worker        .p2align 3
354*4bdc9457SAndroid Build Coastguard Worker6:
355*4bdc9457SAndroid Build Coastguard Worker        TBZ     x1, 2, 7f
356*4bdc9457SAndroid Build Coastguard Worker        ST1     {v0.s}[2], [x7], 4
357*4bdc9457SAndroid Build Coastguard Worker        STR     s0, [x6], 4
358*4bdc9457SAndroid Build Coastguard Worker        EXT     v0.16b, v0.16b, v0.16b, 4
359*4bdc9457SAndroid Build Coastguard Worker
360*4bdc9457SAndroid Build Coastguard Worker7:
361*4bdc9457SAndroid Build Coastguard Worker        TBZ     x1, 1, 8f
362*4bdc9457SAndroid Build Coastguard Worker        ST1     {v0.h}[4], [x7], 2
363*4bdc9457SAndroid Build Coastguard Worker        STR     h0, [x6], 2
364*4bdc9457SAndroid Build Coastguard Worker        EXT     v0.16b, v0.16b, v0.16b, 2
365*4bdc9457SAndroid Build Coastguard Worker8:
366*4bdc9457SAndroid Build Coastguard Worker        TBZ     x1, 0, 9f
367*4bdc9457SAndroid Build Coastguard Worker        ST1     {v0.b}[8], [x7]
368*4bdc9457SAndroid Build Coastguard Worker        STR     b0, [x6]
369*4bdc9457SAndroid Build Coastguard Worker9:
370*4bdc9457SAndroid Build Coastguard Worker        # Restore d8-d15 from stack
371*4bdc9457SAndroid Build Coastguard Worker        LDP     d14, d15, [sp, 48]
372*4bdc9457SAndroid Build Coastguard Worker        LDP     d12, d13, [sp, 32]
373*4bdc9457SAndroid Build Coastguard Worker        LDP     d10, d11, [sp, 16]
374*4bdc9457SAndroid Build Coastguard Worker        LDP     d8, d9, [sp], 64
375*4bdc9457SAndroid Build Coastguard Worker        RET
376*4bdc9457SAndroid Build Coastguard Worker
377*4bdc9457SAndroid Build Coastguard WorkerEND_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_2x8c8__aarch64_neon_mlal_prfm
378*4bdc9457SAndroid Build Coastguard Worker
379*4bdc9457SAndroid Build Coastguard Worker#ifdef __ELF__
380*4bdc9457SAndroid Build Coastguard Worker.section ".note.GNU-stack","",%progbits
381*4bdc9457SAndroid Build Coastguard Worker#endif
382