1 /*
2  * Copyright (c) 2021 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22  * IN THE SOFTWARE.
23  */
24 #ifdef ARM_COMPUTE_ENABLE_SVE
25 
26 #include <cstddef>
27 
28 namespace arm_gemm {
29 
sve_interleaved_fp32_mla_8x3VL_a64fx(const float * Apanel,const float * Bpanel,float * Cpanel,int ablocks,int bblocks,int K)30 void sve_interleaved_fp32_mla_8x3VL_a64fx(
31     const float *Apanel, const float *Bpanel,
32     float *Cpanel, int ablocks, int bblocks, int K) {
33 
34     struct KernelArgs {
35         size_t bblocks = {};
36         size_t K = {};
37         const float *Bpanel = {};
38     } ka;
39 
40     ka.bblocks = bblocks;
41     ka.K = (K/1) - 1;
42     ka.Bpanel = Bpanel;
43 
44     __asm__ __volatile__(
45       "ptrue p0.b\n"
46       "1:"  // Height loop
47       "ldr x22, [%x[args_ptr], %[offsetof_bblocks]]\n"
48       "mov x21, %x[Apanel]\n"
49       "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n"
50       "2:"  // Width loop
51       "ldr x19, [%x[args_ptr], %[offsetof_K]]\n"
52       "mov %x[Apanel], x21\n"
53       "cmp x19, #0x2\n"
54       "mov z8.b, #0x0\n"
55       "mov z9.b, #0x0\n"
56       "ld1w { z0.s }, p0/Z, [x20]\n"
57       "mov z10.b, #0x0\n"
58       "mov z11.b, #0x0\n"
59       "ld1w { z1.s }, p0/Z, [x20, #1, MUL VL]\n"
60       "mov z12.b, #0x0\n"
61       "mov z13.b, #0x0\n"
62       "ld1w { z2.s }, p0/Z, [x20, #2, MUL VL]\n"
63       "mov z14.b, #0x0\n"
64       "mov z15.b, #0x0\n"
65       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
66       "mov z16.b, #0x0\n"
67       "mov z17.b, #0x0\n"
68       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
69       "mov z18.b, #0x0\n"
70       "mov z19.b, #0x0\n"
71       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
72       "mov z20.b, #0x0\n"
73       "mov z21.b, #0x0\n"
74       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
75       "mov z22.b, #0x0\n"
76       "mov z23.b, #0x0\n"
77       "mov z24.b, #0x0\n"
78       "mov z25.b, #0x0\n"
79       "mov z26.b, #0x0\n"
80       "mov z27.b, #0x0\n"
81       "mov z28.b, #0x0\n"
82       "mov z29.b, #0x0\n"
83       "mov z30.b, #0x0\n"
84       "mov z31.b, #0x0\n"
85       "blt 4f\n"
86       "3:"  // main loop head
87       "fmla z8.s, p0/M, z0.s, z3.s\n"
88       "fmla z9.s, p0/M, z1.s, z3.s\n"
89       "sub x19, x19, #0x2\n"
90       "fmla z10.s, p0/M, z2.s, z3.s\n"
91       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
92       "fmla z11.s, p0/M, z0.s, z4.s\n"
93       "fmla z12.s, p0/M, z1.s, z4.s\n"
94       "fmla z13.s, p0/M, z2.s, z4.s\n"
95       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
96       "fmla z14.s, p0/M, z0.s, z5.s\n"
97       "fmla z15.s, p0/M, z1.s, z5.s\n"
98       "cmp x19, #0x2\n"
99       "fmla z16.s, p0/M, z2.s, z5.s\n"
100       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
101       "fmla z17.s, p0/M, z0.s, z6.s\n"
102       "fmla z18.s, p0/M, z1.s, z6.s\n"
103       "fmla z19.s, p0/M, z2.s, z6.s\n"
104       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
105       "fmla z20.s, p0/M, z0.s, z3.s\n"
106       "fmla z21.s, p0/M, z1.s, z3.s\n"
107       "fmla z22.s, p0/M, z2.s, z3.s\n"
108       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #32]\n"
109       "fmla z23.s, p0/M, z0.s, z4.s\n"
110       "fmla z24.s, p0/M, z1.s, z4.s\n"
111       "fmla z25.s, p0/M, z2.s, z4.s\n"
112       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #36]\n"
113       "fmla z26.s, p0/M, z0.s, z5.s\n"
114       "fmla z27.s, p0/M, z1.s, z5.s\n"
115       "fmla z28.s, p0/M, z2.s, z5.s\n"
116       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n"
117       "fmla z29.s, p0/M, z0.s, z6.s\n"
118       "ld1w { z0.s }, p0/Z, [x20, #3, MUL VL]\n"
119       "fmla z30.s, p0/M, z1.s, z6.s\n"
120       "fmla z31.s, p0/M, z2.s, z6.s\n"
121       "ld1w { z1.s }, p0/Z, [x20, #4, MUL VL]\n"
122       "ld1w { z2.s }, p0/Z, [x20, #5, MUL VL]\n"
123       "fmla z8.s, p0/M, z0.s, z3.s\n"
124       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n"
125       "fmla z9.s, p0/M, z1.s, z3.s\n"
126       "fmla z10.s, p0/M, z2.s, z3.s\n"
127       "fmla z11.s, p0/M, z0.s, z4.s\n"
128       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #48]\n"
129       "fmla z12.s, p0/M, z1.s, z4.s\n"
130       "fmla z13.s, p0/M, z2.s, z4.s\n"
131       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n"
132       "fmla z14.s, p0/M, z0.s, z5.s\n"
133       "fmla z15.s, p0/M, z1.s, z5.s\n"
134       "addvl x20, x20, #6\n"
135       "fmla z16.s, p0/M, z2.s, z5.s\n"
136       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n"
137       "fmla z17.s, p0/M, z0.s, z6.s\n"
138       "fmla z18.s, p0/M, z1.s, z6.s\n"
139       "fmla z19.s, p0/M, z2.s, z6.s\n"
140       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #60]\n"
141       "add %x[Apanel], %x[Apanel], #0x40\n"
142       "fmla z20.s, p0/M, z0.s, z3.s\n"
143       "fmla z21.s, p0/M, z1.s, z3.s\n"
144       "fmla z22.s, p0/M, z2.s, z3.s\n"
145       "fmla z23.s, p0/M, z0.s, z4.s\n"
146       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
147       "fmla z24.s, p0/M, z1.s, z4.s\n"
148       "fmla z25.s, p0/M, z2.s, z4.s\n"
149       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
150       "fmla z26.s, p0/M, z0.s, z5.s\n"
151       "fmla z27.s, p0/M, z1.s, z5.s\n"
152       "fmla z28.s, p0/M, z2.s, z5.s\n"
153       "fmla z29.s, p0/M, z0.s, z6.s\n"
154       "ld1w { z0.s }, p0/Z, [x20]\n"
155       "fmla z30.s, p0/M, z1.s, z6.s\n"
156       "fmla z31.s, p0/M, z2.s, z6.s\n"
157       "ld1w { z1.s }, p0/Z, [x20, #1, MUL VL]\n"
158       "ld1w { z2.s }, p0/Z, [x20, #2, MUL VL]\n"
159       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
160       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
161       "bge 3b\n"
162       "4:"  // main loop skip
163       "fmla z8.s, p0/M, z0.s, z3.s\n"
164       "fmla z9.s, p0/M, z1.s, z3.s\n"
165       "addvl x20, x20, #3\n"
166       "fmla z10.s, p0/M, z2.s, z3.s\n"
167       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
168       "fmla z11.s, p0/M, z0.s, z4.s\n"
169       "fmla z12.s, p0/M, z1.s, z4.s\n"
170       "fmla z13.s, p0/M, z2.s, z4.s\n"
171       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
172       "fmla z14.s, p0/M, z0.s, z5.s\n"
173       "fmla z15.s, p0/M, z1.s, z5.s\n"
174       "fmla z16.s, p0/M, z2.s, z5.s\n"
175       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
176       "fmla z17.s, p0/M, z0.s, z6.s\n"
177       "fmla z18.s, p0/M, z1.s, z6.s\n"
178       "fmla z19.s, p0/M, z2.s, z6.s\n"
179       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
180       "fmla z20.s, p0/M, z0.s, z3.s\n"
181       "fmla z21.s, p0/M, z1.s, z3.s\n"
182       "add %x[Apanel], %x[Apanel], #0x20\n"
183       "fmla z22.s, p0/M, z2.s, z3.s\n"
184       "fmla z23.s, p0/M, z0.s, z4.s\n"
185       "fmla z24.s, p0/M, z1.s, z4.s\n"
186       "fmla z25.s, p0/M, z2.s, z4.s\n"
187       "fmla z26.s, p0/M, z0.s, z5.s\n"
188       "fmla z27.s, p0/M, z1.s, z5.s\n"
189       "fmla z28.s, p0/M, z2.s, z5.s\n"
190       "fmla z29.s, p0/M, z0.s, z6.s\n"
191       "fmla z30.s, p0/M, z1.s, z6.s\n"
192       "fmla z31.s, p0/M, z2.s, z6.s\n"
193       "cbz x19, 5f\n"
194       "ld1w { z0.s }, p0/Z, [x20]\n"
195       "ld1w { z1.s }, p0/Z, [x20, #1, MUL VL]\n"
196       "ld1w { z2.s }, p0/Z, [x20, #2, MUL VL]\n"
197       "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n"
198       "fmla z8.s, p0/M, z0.s, z3.s\n"
199       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n"
200       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n"
201       "fmla z9.s, p0/M, z1.s, z3.s\n"
202       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n"
203       "fmla z10.s, p0/M, z2.s, z3.s\n"
204       "fmla z11.s, p0/M, z0.s, z4.s\n"
205       "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n"
206       "fmla z12.s, p0/M, z1.s, z4.s\n"
207       "fmla z13.s, p0/M, z2.s, z4.s\n"
208       "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n"
209       "fmla z14.s, p0/M, z0.s, z5.s\n"
210       "fmla z15.s, p0/M, z1.s, z5.s\n"
211       "fmla z16.s, p0/M, z2.s, z5.s\n"
212       "fmla z17.s, p0/M, z0.s, z6.s\n"
213       "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n"
214       "fmla z18.s, p0/M, z1.s, z6.s\n"
215       "fmla z19.s, p0/M, z2.s, z6.s\n"
216       "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n"
217       "addvl x20, x20, #3\n"
218       "fmla z20.s, p0/M, z0.s, z3.s\n"
219       "fmla z21.s, p0/M, z1.s, z3.s\n"
220       "add %x[Apanel], %x[Apanel], #0x20\n"
221       "fmla z22.s, p0/M, z2.s, z3.s\n"
222       "fmla z23.s, p0/M, z0.s, z4.s\n"
223       "fmla z24.s, p0/M, z1.s, z4.s\n"
224       "fmla z25.s, p0/M, z2.s, z4.s\n"
225       "fmla z26.s, p0/M, z0.s, z5.s\n"
226       "fmla z27.s, p0/M, z1.s, z5.s\n"
227       "fmla z28.s, p0/M, z2.s, z5.s\n"
228       "fmla z29.s, p0/M, z0.s, z6.s\n"
229       "fmla z30.s, p0/M, z1.s, z6.s\n"
230       "fmla z31.s, p0/M, z2.s, z6.s\n"
231       "5:"  // multiply loop done
232       "st1w { z8.s }, p0, [%x[Cpanel]]\n"
233       "subs x22, x22, #0x1\n"
234       "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
235       "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
236       "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
237       "st1w { z12.s }, p0, [%x[Cpanel], #4, MUL VL]\n"
238       "st1w { z13.s }, p0, [%x[Cpanel], #5, MUL VL]\n"
239       "st1w { z14.s }, p0, [%x[Cpanel], #6, MUL VL]\n"
240       "st1w { z15.s }, p0, [%x[Cpanel], #7, MUL VL]\n"
241       "addvl %x[Cpanel], %x[Cpanel], #16\n"
242       "st1w { z16.s }, p0, [%x[Cpanel], #-8, MUL VL]\n"
243       "st1w { z17.s }, p0, [%x[Cpanel], #-7, MUL VL]\n"
244       "st1w { z18.s }, p0, [%x[Cpanel], #-6, MUL VL]\n"
245       "st1w { z19.s }, p0, [%x[Cpanel], #-5, MUL VL]\n"
246       "st1w { z20.s }, p0, [%x[Cpanel], #-4, MUL VL]\n"
247       "st1w { z21.s }, p0, [%x[Cpanel], #-3, MUL VL]\n"
248       "st1w { z22.s }, p0, [%x[Cpanel], #-2, MUL VL]\n"
249       "st1w { z23.s }, p0, [%x[Cpanel], #-1, MUL VL]\n"
250       "st1w { z24.s }, p0, [%x[Cpanel]]\n"
251       "st1w { z25.s }, p0, [%x[Cpanel], #1, MUL VL]\n"
252       "st1w { z26.s }, p0, [%x[Cpanel], #2, MUL VL]\n"
253       "st1w { z27.s }, p0, [%x[Cpanel], #3, MUL VL]\n"
254       "st1w { z28.s }, p0, [%x[Cpanel], #4, MUL VL]\n"
255       "st1w { z29.s }, p0, [%x[Cpanel], #5, MUL VL]\n"
256       "st1w { z30.s }, p0, [%x[Cpanel], #6, MUL VL]\n"
257       "st1w { z31.s }, p0, [%x[Cpanel], #7, MUL VL]\n"
258       "addvl %x[Cpanel], %x[Cpanel], #8\n"
259       "bgt 2b\n"
260       "subs %x[ablocks], %x[ablocks], #0x1\n"
261       "bne 1b\n"
262       : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks)
263       : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks))
264       : "cc", "memory", "p0", "x19", "x20", "x21", "x22", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
265     );
266 }
267 
268 } // namespace arm_gemm
269 #endif // ARM_COMPUTE_ENABLE_SVE
270