1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6#include <xnnpack/assembly.h> 7 8# void xnn_f32_igemm_minmax_ukernel_4x2__aarch64_neonfma${"_prfm" if PREFETCH else ""}_cortex_a75( 9# size_t mr, x0 10# size_t nc, x1 11# size_t kc, x2 / x0 12# size_t ks, x3 / x9 13# const float**restrict a, x4 14# const float*restrict w, x5 15# float*restrict c, x6 16# size_t cm_stride, x7 17# size_t cn_stride, [sp] -> x10 18# size_t a_offset, [sp + 8] -> x11 19# const float* zero, [sp + 16] -> x12 20# const xnn_f32_minmax_params params [sp + 24] -> x8 21 22# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS. 23 24# Register usage 25# A0 x20 v0 v4 26# A1 x13 v1 v5 27# A2 x14 v2 v6 28# A3 x15 v3 v7 29 30# B x5 v16 v17 v18 v19 v20 v21 v22 v23 31 32# C0 x6 v24 v25 33# C1 x16 v26 v27 34# C2 x17 v28 v29 35# C3 x7 v30 v31 36 37# Clamp v4 v5 38 39 40BEGIN_FUNCTION xnn_f32_igemm_minmax_ukernel_4x2__aarch64_neonfma${"_prfm" if PREFETCH else ""}_cortex_a75 41 42 # Load cn_stride, a_offset 43 LDP x10, x11, [sp] 44 45 # Load zero, params pointer 46 LDP x12, x8, [sp, 16] 47 48 # Load min/max values 49 LD2R {v4.2s, v5.2s}, [x8] 50 51 # Save x20 on stack 52 STR x20, [sp, -16]! 53 54 # Clamp C pointers 55 CMP x0, 2 // if mr < 2 56 ADD x16, x6, x7 // c1 = c0 + cm_stride 57 CSEL x16, x6, x16, LO // c1 = c0 58 59 ADD x17, x16, x7 // c2 = c1 + cm_stride 60 // if mr <= 2 61 CSEL x17, x16, x17, LS // c2 = c1 62 63 CMP x0, 4 // if mr < 4 64 ADD x7, x17, x7 // c3 = c2 + cm_stride 65 CSEL x7, x17, x7, LO // c3 = c2 66 670: 68 # Load initial bias from w into accumulators 69 LDR d24, [x5], 8 70 MOV v26.8b, v24.8b 71 MOV v28.8b, v24.8b 72 MOV v30.8b, v24.8b 73 MOVI v25.2s, 0 74 $if PREFETCH: 75 PRFM PLDL1KEEP, [x5, 64] 76 MOVI v27.2s, 0 77 $if PREFETCH: 78 PRFM PLDL1KEEP, [x5, 128] 79 MOVI v29.2s, 0 80 $if PREFETCH: 81 PRFM PLDL1KEEP, [x5, 192] 82 MOVI v31.2s, 0 83 $if PREFETCH: 84 PRFM PLDL1KEEP, [x5, 256] 85 86 MOV x9, x3 // p = ks 87 881: 89 # Load next 4 A pointers 90 LDP x20, x13, [x4], 16 91 LDP x14, x15, [x4], 16 92 93 CMP x20, x12 // if a0 == zero 94 ADD x20, x20, x11 // a0 += a_offset 95 CSEL x20, x12, x20, EQ // a0 = zero, else += a0 + a_offset 96 CMP x13, x12 // if a1 == zero 97 ADD x13, x13, x11 // a1 += a_offset 98 CSEL x13, x12, x13, EQ // a1 = zero, else += a1 + a_offset 99 CMP x14, x12 // if a2 == zero 100 ADD x14, x14, x11 // a2 += a_offset 101 CSEL x14, x12, x14, EQ // a2 = zero, else += a2 + a_offset 102 CMP x15, x12 // if a3 == zero 103 ADD x15, x15, x11 // a3 += a_offset 104 CSEL x15, x12, x15, EQ // a3 = zero, else += a3 + a_offset 105 106 # Is there at least 8 floats (32 bytes) for prologue + epilogue? 107 SUBS x0, x2, 32 // k = kc - 32 108 B.LO 5f 109 110 # Prologue 111 # Read first block of 4 A and B. 112 LDR q0, [x20], 16 113 LDP d20, d21, [x5], 16 114 LDR q1, [x13], 16 115 LDR q2, [x14], 16 116 LDR q3, [x15], 16 117 LDP d22, d23, [x5], 16 118 119 # Is there at least 32. yes do main loop 120 SUBS x0, x0, 32 121 B.LO 3f 122 123 # Main loop - 8 floats of A (32 bytes) 1242: 125 # First block of 4. FMA for first 4, loads for 2nd block of 4. 126 FMLA v24.2s, v20.2s, v0.s[0] 127 LDR q4, [x20], 16 128 FMLA v26.2s, v20.2s, v1.s[0] 129 FMLA v28.2s, v20.2s, v2.s[0] 130 LDR d16, [x5, 0] 131 FMLA v30.2s, v20.2s, v3.s[0] 132 FMLA v25.2s, v21.2s, v0.s[1] 133 LDR q5, [x13], 16 134 FMLA v27.2s, v21.2s, v1.s[1] 135 FMLA v29.2s, v21.2s, v2.s[1] 136 LDR q6, [x14], 16 137 FMLA v31.2s, v21.2s, v3.s[1] 138 FMLA v24.2s, v22.2s, v0.s[2] 139 LDR q7, [x15], 16 140 FMLA v26.2s, v22.2s, v1.s[2] 141 FMLA v28.2s, v22.2s, v2.s[2] 142 LDR d17, [x5, 8] 143 FMLA v30.2s, v22.2s, v3.s[2] 144 FMLA v25.2s, v23.2s, v0.s[3] 145 LDR d18, [x5, 16] 146 FMLA v27.2s, v23.2s, v1.s[3] 147 FMLA v29.2s, v23.2s, v2.s[3] 148 LDR d19, [x5, 24] 149 FMLA v31.2s, v23.2s, v3.s[3] 150 $if PREFETCH: 151 PRFM PLDL1KEEP, [x5, 320] 152 153 # Second block of 4. FMA for second 4, loads for 1st block of 4. 154 FMLA v24.2s, v16.2s, v4.s[0] 155 LDR q0, [x20], 16 156 FMLA v26.2s, v16.2s, v5.s[0] 157 FMLA v28.2s, v16.2s, v6.s[0] 158 LDR d20, [x5, 32] 159 FMLA v30.2s, v16.2s, v7.s[0] 160 FMLA v25.2s, v17.2s, v4.s[1] 161 LDR q1, [x13], 16 162 FMLA v27.2s, v17.2s, v5.s[1] 163 FMLA v29.2s, v17.2s, v6.s[1] 164 LDR q2, [x14], 16 165 FMLA v31.2s, v17.2s, v7.s[1] 166 FMLA v24.2s, v18.2s, v4.s[2] 167 LDR q3, [x15], 16 168 FMLA v26.2s, v18.2s, v5.s[2] 169 FMLA v28.2s, v18.2s, v6.s[2] 170 LDR d21, [x5, 40] 171 FMLA v30.2s, v18.2s, v7.s[2] 172 SUBS x0, x0, 32 173 FMLA v25.2s, v19.2s, v4.s[3] 174 LDR d22, [x5, 48] 175 FMLA v27.2s, v19.2s, v5.s[3] 176 LDR d23, [x5, 56] 177 FMLA v29.2s, v19.2s, v6.s[3] 178 ADD x5, x5, 64 179 FMLA v31.2s, v19.2s, v7.s[3] 180 B.HS 2b 181 1823: 183 # Epilogue 184 # First block of 4. FMA for first 4, loads for 2nd block of 4. 185 FMLA v24.2s, v20.2s, v0.s[0] 186 LDR q4, [x20], 16 187 FMLA v26.2s, v20.2s, v1.s[0] 188 FMLA v28.2s, v20.2s, v2.s[0] 189 LDR d16, [x5, 0] 190 FMLA v30.2s, v20.2s, v3.s[0] 191 FMLA v25.2s, v21.2s, v0.s[1] 192 LDR q5, [x13], 16 193 FMLA v27.2s, v21.2s, v1.s[1] 194 FMLA v29.2s, v21.2s, v2.s[1] 195 LDR q6, [x14], 16 196 FMLA v31.2s, v21.2s, v3.s[1] 197 FMLA v24.2s, v22.2s, v0.s[2] 198 LDR q7, [x15], 16 199 FMLA v26.2s, v22.2s, v1.s[2] 200 FMLA v28.2s, v22.2s, v2.s[2] 201 LDR d17, [x5, 8] 202 FMLA v30.2s, v22.2s, v3.s[2] 203 FMLA v25.2s, v23.2s, v0.s[3] 204 LDR d18, [x5, 16] 205 FMLA v27.2s, v23.2s, v1.s[3] 206 FMLA v29.2s, v23.2s, v2.s[3] 207 LDR d19, [x5, 24] 208 FMLA v31.2s, v23.2s, v3.s[3] 209 $if PREFETCH: 210 PRFM PLDL1KEEP, [x5, 320] 211 212 # Second block of 4. FMA for second 4, no loads 213 FMLA v24.2s, v16.2s, v4.s[0] 214 FMLA v26.2s, v16.2s, v5.s[0] 215 FMLA v28.2s, v16.2s, v6.s[0] 216 FMLA v30.2s, v16.2s, v7.s[0] 217 FMLA v25.2s, v17.2s, v4.s[1] 218 FMLA v27.2s, v17.2s, v5.s[1] 219 FMLA v29.2s, v17.2s, v6.s[1] 220 FMLA v31.2s, v17.2s, v7.s[1] 221 FMLA v24.2s, v18.2s, v4.s[2] 222 FMLA v26.2s, v18.2s, v5.s[2] 223 FMLA v28.2s, v18.2s, v6.s[2] 224 ADDS x0, x0, 32 225 FMLA v30.2s, v18.2s, v7.s[2] 226 FMLA v25.2s, v19.2s, v4.s[3] 227 ADD x5, x5, 32 228 FMLA v27.2s, v19.2s, v5.s[3] 229 FMLA v29.2s, v19.2s, v6.s[3] 230 LD2R {v4.2s, v5.2s}, [x8] // Load min/max values 231 FMLA v31.2s, v19.2s, v7.s[3] 232 233 # Is there a remainder? up to 8 floats (32 bytes) 234 B.NE 5f 235 2364: 237 # ks loop 238 SUBS x9, x9, 32 // ks -= MR * sizeof(void*) 239 B.HI 1b 240 241 FADD v24.2s, v24.2s, v25.2s 242 FADD v26.2s, v26.2s, v27.2s 243 FADD v28.2s, v28.2s, v29.2s 244 FADD v30.2s, v30.2s, v31.2s 245 246 # Clamp 247 FMAX v24.2s, v24.2s, v4.2s 248 FMAX v26.2s, v26.2s, v4.2s 249 FMAX v28.2s, v28.2s, v4.2s 250 FMAX v30.2s, v30.2s, v4.2s 251 SUBS x1, x1, 2 252 FMIN v24.2s, v24.2s, v5.2s 253 FMIN v26.2s, v26.2s, v5.2s 254 FMIN v28.2s, v28.2s, v5.2s 255 FMIN v30.2s, v30.2s, v5.2s 256 257 # Store full 4 x 2 258 B.LO 8f 259 260 STR d30, [x7] 261 ADD x7, x7, x10 262 STR d28, [x17] 263 ADD x17, x17, x10 264 STR d26, [x16] 265 ADD x16, x16, x10 266 STR d24, [x6] 267 ADD x6, x6, x10 268 SUB x4, x4, x3 // a -= ks 269 270 # nc loop 271 B.HI 0b 272 273 # Restore x20 from stack 274 LDR x20, [sp], 16 275 RET 276 2775: 278 # Remainder- 4 floats of A (16 bytes) 279 TBZ x0, 4, 6f 280 281 LDR q0, [x20], 16 282 LDP d20, d21, [x5], 16 283 LDR q1, [x13], 16 284 LDR q2, [x14], 16 285 LDR q3, [x15], 16 286 LDP d22, d23, [x5], 16 287 FMLA v24.2s, v20.2s, v0.s[0] 288 FMLA v26.2s, v20.2s, v1.s[0] 289 FMLA v28.2s, v20.2s, v2.s[0] 290 FMLA v30.2s, v20.2s, v3.s[0] 291 FMLA v25.2s, v21.2s, v0.s[1] 292 FMLA v27.2s, v21.2s, v1.s[1] 293 FMLA v29.2s, v21.2s, v2.s[1] 294 FMLA v31.2s, v21.2s, v3.s[1] 295 FMLA v24.2s, v22.2s, v0.s[2] 296 FMLA v26.2s, v22.2s, v1.s[2] 297 FMLA v28.2s, v22.2s, v2.s[2] 298 FMLA v30.2s, v22.2s, v3.s[2] 299 FMLA v25.2s, v23.2s, v0.s[3] 300 FMLA v27.2s, v23.2s, v1.s[3] 301 FMLA v29.2s, v23.2s, v2.s[3] 302 FMLA v31.2s, v23.2s, v3.s[3] 303 3046: 305 # Remainder- 2 floats of A (8 bytes) 306 TBZ x0, 3, 7f 307 308 LDR d0, [x20], 8 309 LDP d20, d21, [x5], 16 310 LDR d1, [x13], 8 311 LDR d2, [x14], 8 312 LDR d3, [x15], 8 313 FMLA v24.2s, v20.2s, v0.s[0] 314 FMLA v26.2s, v20.2s, v1.s[0] 315 FMLA v28.2s, v20.2s, v2.s[0] 316 FMLA v30.2s, v20.2s, v3.s[0] 317 FMLA v25.2s, v21.2s, v0.s[1] 318 FMLA v27.2s, v21.2s, v1.s[1] 319 FMLA v29.2s, v21.2s, v2.s[1] 320 FMLA v31.2s, v21.2s, v3.s[1] 321 3227: 323 # Remainder- 1 float of A (4 bytes) 324 TBZ x0, 2, 4b 325 326 LDR s0, [x20], 4 327 LDR d20, [x5], 8 328 LDR s1, [x13], 4 329 LDR s2, [x14], 4 330 LDR s3, [x15], 4 331 FMLA v24.2s, v20.2s, v0.s[0] 332 FMLA v26.2s, v20.2s, v1.s[0] 333 FMLA v28.2s, v20.2s, v2.s[0] 334 FMLA v30.2s, v20.2s, v3.s[0] 335 B 4b 336 337 # Store odd width 3388: 339 STR s30, [x7] 340 STR s28, [x17] 341 STR s26, [x16] 342 STR s24, [x6] 343 344 # Restore x20 from stack 345 LDR x20, [sp], 16 346 RET 347 348END_FUNCTION xnn_f32_igemm_minmax_ukernel_4x2__aarch64_neonfma${"_prfm" if PREFETCH else ""}_cortex_a75 349 350#ifdef __ELF__ 351.section ".note.GNU-stack","",%progbits 352#endif 353