1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 7$assert NR % 8 == 0 8$assert 8 <= NR <= 16 9$assert REQUANTIZATION in ["FP32", "RNDNU"] 10$assert not CHANNELWISE or REQUANTIZATION == "FP32" 11#include <assert.h> 12 13#include <arm_neon.h> 14 15#include <xnnpack/igemm.h> 16$if REQUANTIZATION == "FP32": 17 #include <xnnpack/intrinsics-polyfill.h> 18#include <xnnpack/math.h> 19 20 21$DATATYPE = "qc8" if CHANNELWISE else "qs8" 22$PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if REQUANTIZATION == "FP32" else "neon") 23$PARAMS_UNION = "xnn_%s_conv_minmax_params" % DATATYPE.lower() 24void xnn_${DATATYPE}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x${NR}c4__neondot( 25 size_t mr, 26 size_t nc, 27 size_t kc, 28 size_t ks, 29 const int8_t** restrict a, 30 const void* restrict w, 31 int8_t* restrict c, 32 size_t cm_stride, 33 size_t cn_stride, 34 size_t a_offset, 35 const int8_t* zero, 36 const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 37{ 38 assert(mr != 0); 39 assert(mr <= ${MR}); 40 assert(nc != 0); 41 assert(kc != 0); 42 assert(ks != 0); 43 assert(ks % (${MR} * sizeof(void*)) == 0); 44 assert(a_offset % sizeof(int8_t) == 0); 45 assert(a != NULL); 46 assert(w != NULL); 47 assert(c != NULL); 48 49 kc = round_up_po2(kc, 4 * sizeof(int8_t)); 50 int8_t* c0 = c; 51 $for M in range(1, MR): 52 int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride); 53 $if M % 2 == 0: 54 if XNN_UNPREDICTABLE(mr <= ${M}) { 55 c${M} = c${M-1}; 56 } 57 $elif M + 1 == MR: 58 if XNN_UNPREDICTABLE(mr != ${M+1}) { 59 c${M} = c${M-1}; 60 } 61 $else: 62 if XNN_UNPREDICTABLE(mr < ${M+1}) { 63 c${M} = c${M-1}; 64 } 65 66 do { 67 $for N in range(0, NR, 4): 68 int32x4_t vacc0x${ABC[N:N+4]} = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4); 69 $for M in range(1, MR): 70 $for N in range(0, NR, 4): 71 int32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]}; 72 73 size_t p = ks; 74 do { 75 $for M in range(MR): 76 const int8_t* restrict a${M} = a[${M}]; 77 if XNN_UNPREDICTABLE(a${M} != zero) { 78 a${M} = (const int8_t*) ((uintptr_t) a${M} + a_offset); 79 } 80 a += ${MR}; 81 82 // Inner accumulation loop along the ${NR} columns. 83 size_t k = kc; 84 // 2x partial unrolled loop to load 8 bytes at a time. 85 while (k >= 8 * sizeof(int8_t)) { 86 // Load a ${MR}x8 block of activations. 87 $for M in range(MR): 88 const int8x8_t va${M}x01234567 = vld1_s8(a${M}); a${M} += 8; 89 90 // Load a 8x${NR} block of weights. 91 $for K in range(0, 8, 4): 92 $for N in range(0, NR, 4): 93 const int8x16_t vb${ABC[K:K+4]}x${ABC[N:N+4]} = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); 94 95 // Multiply-accumulate: ${MR}x8 * 8x${NR} --> ${MR}x${NR}. 96 $for K in range(0, 8, 4): 97 $for M in range(MR): 98 $for N in range(0, NR, 4): 99 vacc${M}x${ABC[N:N+4]} = vdotq_lane_s32(vacc${M}x${ABC[N:N+4]}, vb${ABC[K:K+4]}x${ABC[N:N+4]}, va${M}x01234567, ${K//4}); 100 101 k -= 8 * sizeof(int8_t); 102 } 103 // Handle up to 4 final positions of `k` 104 if XNN_UNLIKELY(k != 0) { 105 // Load a ${MR}x4 block of activations. 106 $for M in range(MR): 107 const int8x8_t va${M}x01234567 = vld1_s8(a${M}); 108 109 // Load a 4x${NR} block of weights. 110 $for N in range(0, NR, 4): 111 const int8x16_t vb0123x${ABC[N:N+4]} = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); 112 113 // Multiply-accumulate: ${MR}x4 * 4x${NR} --> ${MR}x${NR}. 114 $for M in range(MR): 115 $for N in range(0, NR, 4): 116 vacc${M}x${ABC[N:N+4]} = vdotq_lane_s32(vacc${M}x${ABC[N:N+4]}, vb0123x${ABC[N:N+4]}, va${M}x01234567, 0); 117 } 118 p -= ${MR} * sizeof(void*); 119 } while (p != 0); 120 121 $if REQUANTIZATION == "RNDNU": 122 const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_pre_shift); 123 const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.multiplier); 124 const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_post_shift); 125 126 $for M in range(MR): 127 $for N in range(0, NR, 4): 128 vacc${M}x${ABC[N:N+4]} = vshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_pre_shift); 129 130 $for M in range(MR): 131 $for N in range(0, NR, 4): 132 vacc${M}x${ABC[N:N+4]} = vqdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier); 133 134 $for M in range(MR): 135 $for N in range(0, NR, 4): 136 vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_post_shift); 137 $elif REQUANTIZATION == "FP32": 138 $for M in range(MR): 139 $for N in range(0, NR, 4): 140 float32x4_t vfpacc${M}x${ABC[N:N+4]} = vcvtq_f32_s32(vacc${M}x${ABC[N:N+4]}); 141 142 $if CHANNELWISE: 143 $for N in range(0, NR, 4): 144 const float32x4_t vscale${ABC[N:N+4]} = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4); 145 $for M in range(MR): 146 vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale${ABC[N:N+4]}); 147 $else: 148 const float32x4_t vscale = vld1q_dup_f32(¶ms->${PARAMS_STRUCT}.scale); 149 $for M in range(MR): 150 $for N in range(0, NR, 4): 151 vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale); 152 153 $for M in range(MR): 154 $for N in range(0, NR, 4): 155 vacc${M}x${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${M}x${ABC[N:N+4]}); 156 157 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->${PARAMS_STRUCT}.output_zero_point); 158#if XNN_ARCH_ARM64 159 $for M in range(MR): 160 $for N in range(0, NR, 8): 161 const int16x8_t vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]}), voutput_zero_point); 162 163 $for M in range(MR): 164 $for N in range(0, NR, 16): 165 $if N + 8 < NR: 166 int8x16_t vout${M}x${ABC[N:N+16]} = vqmovn_high_s16(vqmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]}); 167 $elif M % 2 == 1: 168 int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vqmovn_high_s16(vqmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]}); 169 $elif M + 1 == MR: 170 int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]}); 171#else 172 $for M in range(MR): 173 $for N in range(0, NR, 8): 174 const int16x8_t vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vqmovn_s32(vacc${M}x${ABC[N+4:N+8]})), voutput_zero_point); 175 176 $for M in range(MR): 177 $for N in range(0, NR, 16): 178 $if N + 8 < NR: 179 int8x16_t vout${M}x${ABC[N:N+16]} = vcombine_s8(vqmovn_s16(vacc${M}x${ABC[N:N+8]}), vqmovn_s16(vacc${M}x${ABC[N+8:N+16]})); 180 $elif M % 2 == 1: 181 int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vcombine_s8(vqmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vqmovn_s16(vacc${M}x${ABC[N:N+8]})); 182 $elif M + 1 == MR: 183 int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]}); 184#endif 185 $if NR == 8 and MR == 1: 186 const int8x8_t voutput_min = vld1_dup_s8(¶ms->${PARAMS_STRUCT}.output_min); 187 const int8x8_t voutput_max = vld1_dup_s8(¶ms->${PARAMS_STRUCT}.output_max); 188 $else: 189 const int8x16_t voutput_min = vld1q_dup_s8(¶ms->${PARAMS_STRUCT}.output_min); 190 const int8x16_t voutput_max = vld1q_dup_s8(¶ms->${PARAMS_STRUCT}.output_max); 191 192 $for M in reversed(range(MR)): 193 $for N in range(0, NR, 16): 194 $if N + 8 < NR: 195 vout${M}x${ABC[N:N+16]} = vmaxq_s8(vout${M}x${ABC[N:N+16]}, voutput_min); 196 $elif M % 2 == 1: 197 vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmaxq_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_min); 198 $elif M + 1 == MR: 199 $if NR == 8 and MR == 1: 200 vout${M}x${ABC[N:N+8]} = vmax_s8(vout${M}x${ABC[N:N+8]}, voutput_min); 201 $else: 202 vout${M}x${ABC[N:N+8]} = vmax_s8(vout${M}x${ABC[N:N+8]}, vget_low_s8(voutput_min)); 203 204 $for M in reversed(range(MR)): 205 $for N in range(0, NR, 16): 206 $if N + 8 < NR: 207 vout${M}x${ABC[N:N+16]} = vminq_s8(vout${M}x${ABC[N:N+16]}, voutput_max); 208 $elif M % 2 == 1: 209 vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vminq_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_max); 210 $elif M + 1 == MR: 211 $if NR == 8 and MR == 1: 212 vout${M}x${ABC[N:N+8]} = vmin_s8(vout${M}x${ABC[N:N+8]}, voutput_max); 213 $else: 214 vout${M}x${ABC[N:N+8]} = vmin_s8(vout${M}x${ABC[N:N+8]}, vget_low_s8(voutput_max)); 215 216 if (nc >= ${NR}) { 217 $for M in reversed(range(MR)): 218 $for N in range(0, NR, 16): 219 $if N + 8 < NR: 220 vst1q_s8(c${M} + ${N}, vout${M}x${ABC[N:N+16]}); 221 $elif M % 2 == 1: 222 vst1_s8(c${M} + ${N}, vget_high_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 223 vst1_s8(c${M-1} + ${N}, vget_low_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 224 $elif M + 1 == MR: 225 vst1_s8(c${M} + ${N}, vout${M}x${ABC[N:N+8]}); 226 227 $for M in reversed(range(MR)): 228 c${M} = (int8_t*) ((uintptr_t) c${M} + cn_stride); 229 230 a = (const int8_t**restrict) ((uintptr_t) a - ks); 231 232 nc -= ${NR}; 233 } else { 234 $if NR == 16: 235 $for M in reversed(range(MR)): 236 $if M % 2 == 1: 237 int8x16_t vout${M-1}x01234567_${M}x01234567 = vcombine_s8(vget_low_s8(vout${M-1}x0123456789ABCDEF), vget_low_s8(vout${M}x0123456789ABCDEF)); 238 $elif M + 1 == MR: 239 int8x8_t vout${M}x01234567 = vget_low_s8(vout${M}x0123456789ABCDEF); 240 if (nc & 8) { 241 $for M in reversed(range(MR)): 242 $if M % 2 == 1: 243 vst1_s8(c${M}, vget_high_s8(vout${M-1}x01234567_${M}x01234567)); c${M} += 8; 244 vst1_s8(c${M-1}, vget_low_s8(vout${M-1}x01234567_${M}x01234567)); c${M-1} += 8; 245 $elif M + 1 == MR: 246 vst1_s8(c${M}, vout${M}x01234567); c${M} += 8; 247 $for M in reversed(range(MR)): 248 $if M % 2 == 1: 249 vout${M-1}x01234567_${M}x01234567 = vcombine_s8(vget_high_s8(vout${M-1}x0123456789ABCDEF), vget_high_s8(vout${M}x0123456789ABCDEF)); 250 $elif M + 1 == MR: 251 vout${M}x01234567 = vget_high_s8(vout${M}x0123456789ABCDEF); 252 } 253 if (nc & 4) { 254 $for M in reversed(range(MR)): 255 $if M % 2 == 1: 256 vst1q_lane_u32((void*) c${M}, vreinterpretq_u32_s8(vout${M-1}x01234567_${M}x01234567), 2); c${M} += 4; 257 vst1q_lane_u32((void*) c${M-1}, vreinterpretq_u32_s8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 4; 258 $elif M + 1 == MR: 259 vst1_lane_u32((void*) c${M}, vreinterpret_u32_s8(vout${M}x01234567), 0); c${M} += 4; 260 $for M in reversed(range(MR)): 261 $if M % 2 == 1: 262 vout${M-1}x01234567_${M}x01234567 = vextq_s8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 4); 263 $elif M + 1 == MR: 264 vout${M}x01234567 = vext_s8(vout${M}x01234567, vout${M}x01234567, 4); 265 } 266 if (nc & 2) { 267 $for M in reversed(range(MR)): 268 $if M % 2 == 1: 269 vst1q_lane_u16((void*) c${M}, vreinterpretq_u16_s8(vout${M-1}x01234567_${M}x01234567), 4); c${M} += 2; 270 vst1q_lane_u16((void*) c${M-1}, vreinterpretq_u16_s8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 2; 271 $elif M + 1 == MR: 272 vst1_lane_u16((void*) c${M}, vreinterpret_u16_s8(vout${M}x01234567), 0); c${M} += 2; 273 $for M in reversed(range(MR)): 274 $if M % 2 == 1: 275 vout${M-1}x01234567_${M}x01234567 = vextq_s8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 2); 276 $elif M + 1 == MR: 277 vout${M}x01234567 = vext_s8(vout${M}x01234567, vout${M}x01234567, 2); 278 } 279 if (nc & 1) { 280 $for M in reversed(range(MR)): 281 $if M % 2 == 1: 282 vst1q_lane_s8(c${M}, vout${M-1}x01234567_${M}x01234567, 8); 283 vst1q_lane_s8(c${M-1}, vout${M-1}x01234567_${M}x01234567, 0); 284 $elif M + 1 == MR: 285 vst1_lane_s8(c${M}, vout${M}x01234567, 0); 286 } 287 288 nc = 0; 289 } 290 } while (nc != 0); 291} 292