1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 7$assert NR % 8 == 0 8$assert 8 <= NR <= 16 9$assert REQUANTIZATION in ["FP32", "RNDNU"] 10$assert not CHANNELWISE or REQUANTIZATION == "FP32" 11$assert DUP in ["DUP", "LD1R", "LD2R", "LD4R"] 12#include <assert.h> 13 14#include <arm_neon.h> 15 16#include <xnnpack/gemm.h> 17$if REQUANTIZATION == "FP32" and ARMV8: 18 #include <xnnpack/intrinsics-polyfill.h> 19#include <xnnpack/math.h> 20 21 22$DATATYPE = "qc8" if CHANNELWISE else "qs8" 23$PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if REQUANTIZATION == "FP32" and ARMV8 else "neon") 24$PARAMS_UNION = "xnn_%s_conv_minmax_params" % DATATYPE.lower() 25$ISA = "neonv8" if ARMV8 else "neon" 26void xnn_${DATATYPE}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x${NR}c4__${ISA}_${"mlal" if MLA else "mull"}_${DUP.lower()}( 27 size_t mr, 28 size_t nc, 29 size_t kc, 30 size_t ks, 31 const int8_t** restrict a, 32 const void* restrict w, 33 int8_t* restrict c, 34 size_t cm_stride, 35 size_t cn_stride, 36 size_t a_offset, 37 const int8_t* zero, 38 const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 39{ 40 assert(mr != 0); 41 assert(mr <= ${MR}); 42 assert(nc != 0); 43 assert(kc != 0); 44 assert(ks != 0); 45 assert(ks % (${MR} * sizeof(void*)) == 0); 46 assert(a_offset % sizeof(int8_t) == 0); 47 assert(a != NULL); 48 assert(w != NULL); 49 assert(c != NULL); 50 51 kc = round_up_po2(kc, 4 * sizeof(int8_t)); 52 int8_t* c0 = c; 53 $for M in range(1, MR): 54 int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride); 55 $if M % 2 == 0: 56 if XNN_UNPREDICTABLE(mr <= ${M}) { 57 c${M} = c${M-1}; 58 } 59 $elif M + 1 == MR: 60 if XNN_UNPREDICTABLE(mr != ${M+1}) { 61 c${M} = c${M-1}; 62 } 63 $else: 64 if XNN_UNPREDICTABLE(mr < ${M+1}) { 65 c${M} = c${M-1}; 66 } 67 68 do { 69 $for N in range(0, NR, 2): 70 int32x4_t vacc0x${ABC[N:N+2]} = vreinterpretq_s32_u64(vmovl_u32(vld1_u32(w))); w = (const void*) ((uintptr_t) w + 2 * sizeof(int32_t)); 71 $for M in range(1, MR): 72 $for N in range(0, NR, 2): 73 int32x4_t vacc${M}x${ABC[N:N+2]} = vacc0x${ABC[N:N+2]}; 74 75 size_t p = ks; 76 do { 77 $for M in range(MR): 78 const int8_t* restrict a${M} = a[${M}]; 79 if XNN_UNPREDICTABLE(a${M} != zero) { 80 a${M} = (const int8_t*) ((uintptr_t) a${M} + a_offset); 81 } 82 a += ${MR}; 83 84 size_t k = kc; 85 86 $if MLA: 87 while (k >= 16 * sizeof(int8_t)) { 88 $for M in range(MR): 89 $if DUP == "LD2R": 90 const int32x2x2_t va${M}x0 = vld2_dup_s32((const void*)a${M}); a${M} += 8; 91 const int32x2x2_t va${M}x1 = vld2_dup_s32((const void*)a${M}); a${M} += 8; 92 $elif DUP == "LD1R": 93 const int32x2_t va${M}0x0 = vld1_dup_s32((const void*)a${M}); 94 const int32x2_t va${M}1x0 = vld1_dup_s32((const void*)(a${M} + 4)); a${M} += 8; 95 const int32x2_t va${M}0x1 = vld1_dup_s32((const void*)a${M}); 96 const int32x2_t va${M}1x1 = vld1_dup_s32((const void*)(a${M} + 4)); a${M} += 8; 97 $else: 98 const int8x8_t va${M}x0 = vld1_s8(a${M}); a${M} += 8; 99 const int8x8_t va${M}x1 = vld1_s8(a${M}); a${M} += 8; 100 101 $for K in range(2): 102 $for N in range(0, NR, 2): 103 const int8x8_t vb${ABC[N:N+2]}c${K}x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t)); 104 105 $for K in range(2): 106 $for M in range(MR): 107 $if DUP == "LD2R": 108 const int8x8_t va${M}c${K}x0 = vreinterpret_s8_s32(va${M}x0.val[${K}]); 109 const int8x8_t va${M}c${K}x1 = vreinterpret_s8_s32(va${M}x1.val[${K}]); 110 $elif DUP == "LD1R": 111 const int8x8_t va${M}c${K}x0 = vreinterpret_s8_s32(va${M}${K}x0); 112 const int8x8_t va${M}c${K}x1 = vreinterpret_s8_s32(va${M}${K}x1); 113 $else: 114 const int8x8_t va${M}c${K}x0 = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(va${M}x0), ${K})); 115 const int8x8_t va${M}c${K}x1 = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(va${M}x1), ${K})); 116 117 $for N in range(0, NR, 2): 118 $for M in range(MR): 119 int16x8_t vprod${M}x${ABC[N:N+2]}c${K} = vmull_s8(vb${ABC[N:N+2]}c${K}x0, va${M}c${K}x0); 120 const int8x8_t vb${ABC[N:N+2]}c${K}x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t)); 121 $for M in range(MR): 122 vprod${M}x${ABC[N:N+2]}c${K} = vmlal_s8(vprod${M}x${ABC[N:N+2]}c${K}, vb${ABC[N:N+2]}c${K}x1, va${M}c${K}x1); 123 $for M in range(MR): 124 vacc${M}x${ABC[N:N+2]} = vpadalq_s16(vacc${M}x${ABC[N:N+2]}, vprod${M}x${ABC[N:N+2]}c${K}); 125 126 k -= 16 * sizeof(int8_t); 127 } 128 129 ${"if" if MLA else "while"} (k >= 8 * sizeof(int8_t)) { 130 $for M in range(MR): 131 $if DUP == "LD2R": 132 const int32x2x2_t va${M} = vld2_dup_s32((const void*)a${M}); a${M} += 8; 133 $elif DUP == "LD1R": 134 const int32x2_t va${M}0 = vld1_dup_s32((const void*)a${M}); 135 const int32x2_t va${M}1 = vld1_dup_s32((const void*)(a${M} + 4)); a${M} += 8; 136 $else: 137 const int8x8_t va${M} = vld1_s8(a${M}); a${M} += 8; 138 139 $for K in range(2): 140 $for N in range(0, NR, 2): 141 const int8x8_t vb${ABC[N:N+2]}c${K} = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t)); 142 143 $for K in range(2): 144 $for M in range(MR): 145 $if DUP == "LD2R": 146 const int8x8_t va${M}c${K} = vreinterpret_s8_s32(va${M}.val[${K}]); 147 $elif DUP == "LD1R": 148 const int8x8_t va${M}c${K} = vreinterpret_s8_s32(va${M}${K}); 149 $else: 150 const int8x8_t va${M}c${K} = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(va${M}), ${K})); 151 152 $for N in range(0, NR, 2): 153 $for M in range(MR): 154 const int16x8_t vprod${M}x${ABC[N:N+2]}c${K} = vmull_s8(vb${ABC[N:N+2]}c${K}, va${M}c${K}); 155 $for M in range(MR): 156 vacc${M}x${ABC[N:N+2]} = vpadalq_s16(vacc${M}x${ABC[N:N+2]}, vprod${M}x${ABC[N:N+2]}c${K}); 157 158 k -= 8 * sizeof(int8_t); 159 } 160 161 if XNN_UNLIKELY(k != 0) { 162 $for M in range(MR): 163 const int8x8_t va${M} = vld1_s8(a${M}); a${M} = (const int8_t*) ((uintptr_t) a${M} + k); 164 165 $for N in range(0, NR, 2): 166 const int8x8_t vb${ABC[N:N+2]}c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t)); 167 168 $for M in range(MR): 169 const int8x8_t va${M}c0 = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(va${M}), 0)); 170 $for N in range(0, NR, 2): 171 const int16x8_t vprod${M}x${ABC[N:N+2]}c0 = vmull_s8(vb${ABC[N:N+2]}c0, va${M}c0); 172 vacc${M}x${ABC[N:N+2]} = vpadalq_s16(vacc${M}x${ABC[N:N+2]}, vprod${M}x${ABC[N:N+2]}c0); 173 } 174 p -= ${MR} * sizeof(void*); 175 } while (p != 0); 176 177#if XNN_ARCH_ARM64 178 $for M in range(MR): 179 $for N in range(0, NR, 4): 180 int32x4_t vacc${M}x${ABC[N:N+4]} = vpaddq_s32(vacc${M}x${ABC[N:N+2]}, vacc${M}x${ABC[N+2:N+4]}); 181#else 182 $for M in range(MR): 183 $for N in range(0, NR, 4): 184 const int32x2_t vsum${M}x${ABC[N:N+2]} = vpadd_s32(vget_low_s32(vacc${M}x${ABC[N:N+2]}), vget_high_s32(vacc${M}x${ABC[N:N+2]})); 185 const int32x2_t vsum${M}x${ABC[N+2:N+4]} = vpadd_s32(vget_low_s32(vacc${M}x${ABC[N+2:N+4]}), vget_high_s32(vacc${M}x${ABC[N+2:N+4]})); 186 int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]}); 187#endif 188 189 $if REQUANTIZATION == "RNDNU": 190 const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_pre_shift); 191 const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.multiplier); 192 const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_post_shift); 193 194 $for M in range(MR): 195 $for N in range(0, NR, 4): 196 vacc${M}x${ABC[N:N+4]} = vqshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_pre_shift); 197 198 $for M in range(MR): 199 $for N in range(0, NR, 4): 200 vacc${M}x${ABC[N:N+4]} = vqdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier); 201 202 $for M in range(MR): 203 $for N in range(0, NR, 4): 204 vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_post_shift); 205 $elif REQUANTIZATION == "FP32": 206 $for M in range(MR): 207 $for N in range(0, NR, 4): 208 float32x4_t vfpacc${M}x${ABC[N:N+4]} = vcvtq_f32_s32(vacc${M}x${ABC[N:N+4]}); 209 210 $if CHANNELWISE: 211 $for N in range(0, NR, 4): 212 const float32x4_t vscale${ABC[N:N+4]} = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4); 213 $for M in range(MR): 214 vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale${ABC[N:N+4]}); 215 $else: 216 const float32x4_t vscale = vld1q_dup_f32(¶ms->${PARAMS_STRUCT}.scale); 217 $for M in range(MR): 218 $for N in range(0, NR, 4): 219 vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale); 220 221 $if ARMV8: 222 $for M in range(MR): 223 $for N in range(0, NR, 4): 224 vacc${M}x${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${M}x${ABC[N:N+4]}); 225 $else: 226 const float32x4_t vmagic_bias = vld1q_dup_f32(¶ms->${PARAMS_STRUCT}.magic_bias); 227 $for M in range(MR): 228 $for N in range(0, NR, 4): 229 vacc${M}x${ABC[N:N+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${M}x${ABC[N:N+4]}, vmagic_bias)); 230 231 const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.magic_bias_less_output_zero_point); 232 $for M in range(MR): 233 $for N in range(0, NR, 4): 234 vacc${M}x${ABC[N:N+4]} = vqsubq_s32(vacc${M}x${ABC[N:N+4]}, vmagic_bias_less_output_zero_point); 235 236 $if REQUANTIZATION != "FP32" or ARMV8: 237 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->${PARAMS_STRUCT}.output_zero_point); 238#if XNN_ARCH_ARM64 239 $for M in range(MR): 240 $for N in range(0, NR, 8): 241 int16x8_t vacc${M}x${ABC[N:N+8]} = vqmovn_high_s32(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]}); 242 243 $if REQUANTIZATION != "FP32" or ARMV8: 244 $for M in range(MR): 245 $for N in range(0, NR, 8): 246 vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vacc${M}x${ABC[N:N+8]}, voutput_zero_point); 247 248 $for M in range(MR): 249 $for N in range(0, NR, 16): 250 $if N + 8 < NR: 251 int8x16_t vout${M}x${ABC[N:N+16]} = vqmovn_high_s16(vqmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]}); 252 $elif M % 2 == 1: 253 int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vqmovn_high_s16(vqmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]}); 254 $elif M + 1 == MR: 255 int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]}); 256#else 257 $for M in range(MR): 258 $for N in range(0, NR, 8): 259 int16x8_t vacc${M}x${ABC[N:N+8]} = vcombine_s16(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vqmovn_s32(vacc${M}x${ABC[N+4:N+8]})); 260 261 $if REQUANTIZATION != "FP32" or ARMV8: 262 $for M in range(MR): 263 $for N in range(0, NR, 8): 264 vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vacc${M}x${ABC[N:N+8]}, voutput_zero_point); 265 266 $for M in range(MR): 267 $for N in range(0, NR, 16): 268 $if N + 8 < NR: 269 int8x16_t vout${M}x${ABC[N:N+16]} = vcombine_s8(vqmovn_s16(vacc${M}x${ABC[N:N+8]}), vqmovn_s16(vacc${M}x${ABC[N+8:N+16]})); 270 $elif M % 2 == 1: 271 int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vcombine_s8(vqmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vqmovn_s16(vacc${M}x${ABC[N:N+8]})); 272 $elif M + 1 == MR: 273 int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]}); 274#endif 275 276 $if NR == 8 and MR == 1: 277 const int8x8_t voutput_min = vld1_dup_s8(¶ms->${PARAMS_STRUCT}.output_min); 278 $else: 279 const int8x16_t voutput_min = vld1q_dup_s8(¶ms->${PARAMS_STRUCT}.output_min); 280 $for M in range(MR): 281 $for N in range(0, NR, 16): 282 $if N + 8 < NR: 283 vout${M}x${ABC[N:N+16]} = vmaxq_s8(vout${M}x${ABC[N:N+16]}, voutput_min); 284 $elif M % 2 == 1: 285 vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmaxq_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_min); 286 $elif M + 1 == MR: 287 $if NR == 8 and MR == 1: 288 vout${M}x${ABC[N:N+8]} = vmax_s8(vout${M}x${ABC[N:N+8]}, voutput_min); 289 $else: 290 vout${M}x${ABC[N:N+8]} = vmax_s8(vout${M}x${ABC[N:N+8]}, vget_low_s8(voutput_min)); 291 292 $if NR == 8 and MR == 1: 293 const int8x8_t voutput_max = vld1_dup_s8(¶ms->${PARAMS_STRUCT}.output_max); 294 $else: 295 const int8x16_t voutput_max = vld1q_dup_s8(¶ms->${PARAMS_STRUCT}.output_max); 296 $for M in range(MR): 297 $for N in range(0, NR, 16): 298 $if N + 8 < NR: 299 vout${M}x${ABC[N:N+16]} = vminq_s8(vout${M}x${ABC[N:N+16]}, voutput_max); 300 $elif M % 2 == 1: 301 vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vminq_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_max); 302 $elif M + 1 == MR: 303 $if NR == 8 and MR == 1: 304 vout${M}x${ABC[N:N+8]} = vmin_s8(vout${M}x${ABC[N:N+8]}, voutput_max); 305 $else: 306 vout${M}x${ABC[N:N+8]} = vmin_s8(vout${M}x${ABC[N:N+8]}, vget_low_s8(voutput_max)); 307 308 if (nc >= ${NR}) { 309 $for M in reversed(range(MR)): 310 $for N in range(0, NR, 16): 311 $if N + 8 < NR: 312 vst1q_s8(c${M} + ${N}, vout${M}x${ABC[N:N+16]}); 313 $elif M % 2 == 1: 314 vst1_s8(c${M} + ${N}, vget_high_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 315 vst1_s8(c${M-1} + ${N}, vget_low_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 316 $elif M + 1 == MR: 317 vst1_s8(c${M} + ${N}, vout${M}x${ABC[N:N+8]}); 318 319 $for M in reversed(range(MR)): 320 c${M} = (int8_t*) ((uintptr_t) c${M} + cn_stride); 321 322 a = (const int8_t**restrict) ((uintptr_t) a - ks); 323 324 nc -= ${NR}; 325 } else { 326 $if NR == 16: 327 $for M in reversed(range(MR)): 328 $if M % 2 == 1: 329 int8x16_t vout${M-1}x01234567_${M}x01234567 = vcombine_s8(vget_low_s8(vout${M-1}x0123456789ABCDEF), vget_low_s8(vout${M}x0123456789ABCDEF)); 330 $elif M + 1 == MR: 331 int8x8_t vout${M}x01234567 = vget_low_s8(vout${M}x0123456789ABCDEF); 332 if (nc & 8) { 333 $for M in reversed(range(MR)): 334 $if M % 2 == 1: 335 vst1_s8(c${M}, vget_high_s8(vout${M-1}x01234567_${M}x01234567)); c${M} += 8; 336 vst1_s8(c${M-1}, vget_low_s8(vout${M-1}x01234567_${M}x01234567)); c${M-1} += 8; 337 $elif M + 1 == MR: 338 vst1_s8(c${M}, vout${M}x01234567); c${M} += 8; 339 $for M in reversed(range(MR)): 340 $if M % 2 == 1: 341 vout${M-1}x01234567_${M}x01234567 = vcombine_s8(vget_high_s8(vout${M-1}x0123456789ABCDEF), vget_high_s8(vout${M}x0123456789ABCDEF)); 342 $elif M + 1 == MR: 343 vout${M}x01234567 = vget_high_s8(vout${M}x0123456789ABCDEF); 344 } 345 if (nc & 4) { 346 $for M in reversed(range(MR)): 347 $if M % 2 == 1: 348 vst1q_lane_u32((void*) c${M}, vreinterpretq_u32_s8(vout${M-1}x01234567_${M}x01234567), 2); c${M} += 4; 349 vst1q_lane_u32((void*) c${M-1}, vreinterpretq_u32_s8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 4; 350 $elif M + 1 == MR: 351 vst1_lane_u32((void*) c${M}, vreinterpret_u32_s8(vout${M}x01234567), 0); c${M} += 4; 352 $for M in reversed(range(MR)): 353 $if M % 2 == 1: 354 vout${M-1}x01234567_${M}x01234567 = vextq_s8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 4); 355 $elif M + 1 == MR: 356 vout${M}x01234567 = vext_s8(vout${M}x01234567, vout${M}x01234567, 4); 357 } 358 if (nc & 2) { 359 $for M in reversed(range(MR)): 360 $if M % 2 == 1: 361 vst1q_lane_u16((void*) c${M}, vreinterpretq_u16_s8(vout${M-1}x01234567_${M}x01234567), 4); c${M} += 2; 362 vst1q_lane_u16((void*) c${M-1}, vreinterpretq_u16_s8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 2; 363 $elif M + 1 == MR: 364 vst1_lane_u16((void*) c${M}, vreinterpret_u16_s8(vout${M}x01234567), 0); c${M} += 2; 365 $for M in reversed(range(MR)): 366 $if M % 2 == 1: 367 vout${M-1}x01234567_${M}x01234567 = vextq_s8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 2); 368 $elif M + 1 == MR: 369 vout${M}x01234567 = vext_s8(vout${M}x01234567, vout${M}x01234567, 2); 370 } 371 if (nc & 1) { 372 $for M in reversed(range(MR)): 373 $if M % 2 == 1: 374 vst1q_lane_s8(c${M}, vout${M-1}x01234567_${M}x01234567, 8); 375 vst1q_lane_s8(c${M-1}, vout${M-1}x01234567_${M}x01234567, 0); 376 $elif M + 1 == MR: 377 vst1_lane_s8(c${M}, vout${M}x01234567, 0); 378 } 379 380 nc = 0; 381 } 382 } while (nc != 0); 383} 384