xref: /aosp_15_r20/external/XNNPACK/src/f32-igemm/gen/1x8-minmax-aarch64-neonfma-cortex-a75.S (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1// Auto-generated file. Do not edit!
2//   Template: src/f32-igemm/1x8-aarch64-neonfma-cortex-a75.S.in
3//   Generator: tools/xngen
4//
5// Copyright 2019 Google LLC
6//
7// This source code is licensed under the BSD-style license found in the
8// LICENSE file in the root directory of this source tree.
9
10#include <xnnpack/assembly.h>
11
12# void xnn_f32_igemm_minmax_ukernel_4x8__aarch64_neonfma_cortex_a75(
13#     size_t mr,                         (x0) - unused.  mr = 1
14#     size_t nc,                         x1
15#     size_t kc,                         x2 / x0
16#     size_t ks,                         x3 / x9
17#     const float**restrict a,           x4
18#     const float*restrict w,            x5
19#     float*restrict c,                  x6
20#     size_t cm_stride,                  (x7) - unused
21#     size_t cn_stride,                  [sp] -> x10
22#     size_t a_offset,                   [sp + 8] -> x11
23#     const float* zero,                 [sp + 16] -> x12
24#     const xnn_f32_minmax_params params [sp + 24] -> (x8)
25
26# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
27
28# A pointer
29# x8  a0
30
31# C pointer
32# x6  c0
33
34BEGIN_FUNCTION xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75
35
36        # Load cn_stride, a_offset
37        LDP     x10, x11, [sp]
38
39        # Load zero, params pointer
40        LDP     x12, x8, [sp, 16]
41
42        # Load min/max values
43        LD2R    {v30.4s, v31.4s}, [x8]
44
450:
46        # Load initial bias from w into accumulators
47        LDP     q16, q17, [x5], 32
48        MOVI    v18.4s, 0               // second set of C for pipelining FMLA
49        MOVI    v19.4s, 0
50
51        MOV     x9, x3                  // p = ks
52
531:
54        # Load next A pointer
55        LDR     x8, [x4], 8
56
57        CMP     x8, x12                 // if a0 == zero
58        ADD     x8, x8, x11             // a0 += a_offset
59        CSEL    x8, x12, x8, EQ         //   a0 = zero, else += a0 + a_offset
60
61        # Is there at least 8 floats (32 bytes) for prologue + epilogue?
62        SUBS    x0, x2, 32              // k = kc - 32
63        B.LO    4f
64
65        # 16 prologue
66        # Read first block of A and B.
67        LDP     q20, q21, [x5], 32
68        LDP     q22, q23, [x5], 32
69        LDP     q24, q25, [x5], 32
70        LDP     q26, q27, [x5], 32
71        LDR     q0, [x8], 16
72
73        # Is there at least 8.  yes do main loop
74        SUBS    x0, x0, 32
75        B.LO    3f
76
77        # Main loop - 8 floats of A (32 bytes)
782:
79        # First block of 4.  FMA for first 4, loads for 2nd block of 4.
80        FMLA    v16.4s, v20.4s, v0.s[0]
81        LDR     q1, [x8], 16
82        FMLA    v17.4s, v21.4s, v0.s[0]
83        LDP     q20, q21, [x5], 32
84        FMLA    v18.4s, v22.4s, v0.s[1]
85        FMLA    v19.4s, v23.4s, v0.s[1]
86        LDP     q22, q23, [x5], 32
87        FMLA    v16.4s, v24.4s, v0.s[2]
88        FMLA    v17.4s, v25.4s, v0.s[2]
89        LDP     q24, q25, [x5], 32
90        FMLA    v18.4s, v26.4s, v0.s[3]
91        FMLA    v19.4s, v27.4s, v0.s[3]
92        LDP     q26, q27, [x5], 32
93
94        # Second block of 4.  FMA for second 4, loads for 1st block of 4.
95        FMLA    v16.4s, v20.4s, v1.s[0]
96        LDR     q0, [x8], 16
97        FMLA    v17.4s, v21.4s, v1.s[0]
98        LDP     q20, q21, [x5], 32
99        FMLA    v18.4s, v22.4s, v1.s[1]
100        FMLA    v19.4s, v23.4s, v1.s[1]
101        LDP     q22, q23, [x5], 32
102        FMLA    v16.4s, v24.4s, v1.s[2]
103        FMLA    v17.4s, v25.4s, v1.s[2]
104        LDP     q24, q25, [x5], 32
105        FMLA    v18.4s, v26.4s, v1.s[3]
106        FMLA    v19.4s, v27.4s, v1.s[3]
107        SUBS    x0, x0, 32
108        LDP     q26, q27, [x5], 32
109        B.HS    2b
110
1113:
112        # Epilogue
113
114        # First block of 4.  FMA for first 4, loads for 2nd block of 4.
115        FMLA    v16.4s, v20.4s, v0.s[0]
116        LDR     q1, [x8], 16
117        FMLA    v17.4s, v21.4s, v0.s[0]
118        LDP     q20, q21, [x5], 32
119        FMLA    v18.4s, v22.4s, v0.s[1]
120        FMLA    v19.4s, v23.4s, v0.s[1]
121        LDP     q22, q23, [x5], 32
122        FMLA    v16.4s, v24.4s, v0.s[2]
123        FMLA    v17.4s, v25.4s, v0.s[2]
124        LDP     q24, q25, [x5], 32
125        FMLA    v18.4s, v26.4s, v0.s[3]
126        FMLA    v19.4s, v27.4s, v0.s[3]
127        LDP     q26, q27, [x5], 32
128
129        # Second block of 4.  no loads
130        FMLA    v16.4s, v20.4s, v1.s[0]
131        FMLA    v17.4s, v21.4s, v1.s[0]
132        FMLA    v18.4s, v22.4s, v1.s[1]
133        FMLA    v19.4s, v23.4s, v1.s[1]
134        FMLA    v16.4s, v24.4s, v1.s[2]
135        FMLA    v17.4s, v25.4s, v1.s[2]
136        FMLA    v18.4s, v26.4s, v1.s[3]
137        FMLA    v19.4s, v27.4s, v1.s[3]
138
1394:
140        # Is there a remainder?- 4 floats of A (16 bytes)
141        TBNZ    x0, 4, 6f
142        # Is there a remainder?- 2 floats of A (8 bytes)
143        TBNZ    x0, 3, 7f
144        # Is there a remainder?- 1 float of A (4 bytes)
145        TBNZ    x0, 2, 9f
146
1475:
148        # ks loop
149        SUBS    x9, x9, 8               // ks -= MR * sizeof(void*)
150        B.HI    1b
151
152        FADD    v16.4s, v16.4s, v18.4s
153        FADD    v17.4s, v17.4s, v19.4s
154
155        # Clamp
156        FMAX    v16.4s, v16.4s, v30.4s
157        FMAX    v17.4s, v17.4s, v30.4s
158        FMIN    v16.4s, v16.4s, v31.4s
159        FMIN    v17.4s, v17.4s, v31.4s
160
161        # Store full 1 x 8
162        SUBS    x1, x1, 8
163        B.LO    10f
164
165        STP     q16, q17, [x6]
166        ADD     x6, x6, x10
167
168        SUB     x4, x4, x3              // a -= ks
169
170        # nc loop
171        B.HI    0b
172
173        RET
174
1756:
176        # Remainder- 4 floats of A (16 bytes)
177        LDP     q20, q21, [x5], 32
178        LDR     q0, [x8], 16
179        FMLA    v16.4s, v20.4s, v0.s[0]
180        FMLA    v17.4s, v21.4s, v0.s[0]
181        LDP     q22, q23, [x5], 32
182        LDP     q24, q25, [x5], 32
183        LDP     q26, q27, [x5], 32
184        FMLA    v18.4s, v22.4s, v0.s[1]
185        FMLA    v19.4s, v23.4s, v0.s[1]
186        FMLA    v16.4s, v24.4s, v0.s[2]
187        FMLA    v17.4s, v25.4s, v0.s[2]
188        FMLA    v18.4s, v26.4s, v0.s[3]
189        FMLA    v19.4s, v27.4s, v0.s[3]
190
191        TBZ     x0, 3, 8f
1927:
193        # Remainder- 2 floats of A (8 bytes)
194        LDP     q20, q21, [x5], 32
195        LDR     d0, [x8], 8
196        FMLA    v16.4s, v20.4s, v0.s[0]
197        FMLA    v17.4s, v21.4s, v0.s[0]
198        LDP     q22, q23, [x5], 32
199        FMLA    v18.4s, v22.4s, v0.s[1]
200        FMLA    v19.4s, v23.4s, v0.s[1]
2018:
202        TBZ     x0, 2, 5b
2039:
204        # Remainder- 1 float of A (4 bytes)
205        LDP     q20, q21, [x5], 32
206        LDR     s0, [x8], 4
207        FMLA    v16.4s, v20.4s, v0.s[0]
208        FMLA    v17.4s, v21.4s, v0.s[0]
209        B       5b
210
21110:
212        # Store odd channels
213        TBZ     x1, 2, 11f
214        STR     q16, [x6], 16
215        MOV     v16.16b, v17.16b
216
21711:
218        TBZ     x1, 1, 12f
219        STR     d16, [x6], 8
220        DUP     d16, v16.d[1]
221
22212:
223        TBZ     x1, 0, 13f
224        STR     s16, [x6], 4
22513:
226        RET
227
228END_FUNCTION xnn_f32_igemm_minmax_ukernel_1x8__aarch64_neonfma_cortex_a75
229
230#ifdef __ELF__
231.section ".note.GNU-stack","",%progbits
232#endif
233