1*4bdc9457SAndroid Build Coastguard Worker// Copyright 2021 Google LLC 2*4bdc9457SAndroid Build Coastguard Worker// 3*4bdc9457SAndroid Build Coastguard Worker// This source code is licensed under the BSD-style license found in the 4*4bdc9457SAndroid Build Coastguard Worker// LICENSE file in the root directory of this source tree. 5*4bdc9457SAndroid Build Coastguard Worker 6*4bdc9457SAndroid Build Coastguard Worker$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 7*4bdc9457SAndroid Build Coastguard Worker$assert NR % 8 == 0 8*4bdc9457SAndroid Build Coastguard Worker$assert 8 <= NR <= 16 9*4bdc9457SAndroid Build Coastguard Worker$assert REQUANTIZATION in ["FP32", "RNDNU"] 10*4bdc9457SAndroid Build Coastguard Worker$assert not CHANNELWISE or REQUANTIZATION == "FP32" 11*4bdc9457SAndroid Build Coastguard Worker#include <assert.h> 12*4bdc9457SAndroid Build Coastguard Worker 13*4bdc9457SAndroid Build Coastguard Worker#include <arm_neon.h> 14*4bdc9457SAndroid Build Coastguard Worker 15*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/igemm.h> 16*4bdc9457SAndroid Build Coastguard Worker$if REQUANTIZATION == "FP32" and ARMV8: 17*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/intrinsics-polyfill.h> 18*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/math.h> 19*4bdc9457SAndroid Build Coastguard Worker 20*4bdc9457SAndroid Build Coastguard Worker 21*4bdc9457SAndroid Build Coastguard Worker$DATATYPE = "qc8" if CHANNELWISE else "qs8" 22*4bdc9457SAndroid Build Coastguard Worker$PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if REQUANTIZATION == "FP32" and ARMV8 else "neon") 23*4bdc9457SAndroid Build Coastguard Worker$PARAMS_UNION = "xnn_%s_conv_minmax_params" % DATATYPE.lower() 24*4bdc9457SAndroid Build Coastguard Worker$ISA = "neonv8" if ARMV8 else "neon" 25*4bdc9457SAndroid Build Coastguard Workervoid xnn_${DATATYPE}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_${MR}x${NR}c8__${ISA}_${"mlal" if MLA else "mull"}( 26*4bdc9457SAndroid Build Coastguard Worker size_t mr, 27*4bdc9457SAndroid Build Coastguard Worker size_t nc, 28*4bdc9457SAndroid Build Coastguard Worker size_t kc, 29*4bdc9457SAndroid Build Coastguard Worker size_t ks, 30*4bdc9457SAndroid Build Coastguard Worker const int8_t** restrict a, 31*4bdc9457SAndroid Build Coastguard Worker const void* restrict w, 32*4bdc9457SAndroid Build Coastguard Worker int8_t* restrict c, 33*4bdc9457SAndroid Build Coastguard Worker size_t cm_stride, 34*4bdc9457SAndroid Build Coastguard Worker size_t cn_stride, 35*4bdc9457SAndroid Build Coastguard Worker size_t a_offset, 36*4bdc9457SAndroid Build Coastguard Worker const int8_t* zero, 37*4bdc9457SAndroid Build Coastguard Worker const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 38*4bdc9457SAndroid Build Coastguard Worker{ 39*4bdc9457SAndroid Build Coastguard Worker assert(mr != 0); 40*4bdc9457SAndroid Build Coastguard Worker assert(mr <= ${MR}); 41*4bdc9457SAndroid Build Coastguard Worker assert(nc != 0); 42*4bdc9457SAndroid Build Coastguard Worker assert(kc != 0); 43*4bdc9457SAndroid Build Coastguard Worker assert(ks != 0); 44*4bdc9457SAndroid Build Coastguard Worker assert(ks % (${MR} * sizeof(void*)) == 0); 45*4bdc9457SAndroid Build Coastguard Worker assert(a_offset % sizeof(int8_t) == 0); 46*4bdc9457SAndroid Build Coastguard Worker assert(a != NULL); 47*4bdc9457SAndroid Build Coastguard Worker assert(w != NULL); 48*4bdc9457SAndroid Build Coastguard Worker assert(c != NULL); 49*4bdc9457SAndroid Build Coastguard Worker 50*4bdc9457SAndroid Build Coastguard Worker kc = round_up_po2(kc, 8 * sizeof(int8_t)); 51*4bdc9457SAndroid Build Coastguard Worker int8_t* c0 = c; 52*4bdc9457SAndroid Build Coastguard Worker $for M in range(1, MR): 53*4bdc9457SAndroid Build Coastguard Worker int8_t* c${M} = (int8_t*) ((uintptr_t) c${M-1} + cm_stride); 54*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 0: 55*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(mr <= ${M}) { 56*4bdc9457SAndroid Build Coastguard Worker c${M} = c${M-1}; 57*4bdc9457SAndroid Build Coastguard Worker } 58*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 59*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(mr != ${M+1}) { 60*4bdc9457SAndroid Build Coastguard Worker c${M} = c${M-1}; 61*4bdc9457SAndroid Build Coastguard Worker } 62*4bdc9457SAndroid Build Coastguard Worker $else: 63*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(mr < ${M+1}) { 64*4bdc9457SAndroid Build Coastguard Worker c${M} = c${M-1}; 65*4bdc9457SAndroid Build Coastguard Worker } 66*4bdc9457SAndroid Build Coastguard Worker 67*4bdc9457SAndroid Build Coastguard Worker do { 68*4bdc9457SAndroid Build Coastguard Worker $for N in range(NR): 69*4bdc9457SAndroid Build Coastguard Worker int32x4_t vacc0x${N} = vld1q_lane_s32(w, vmovq_n_s32(0), 0); w = (const void*) ((uintptr_t) w + sizeof(int32_t)); 70*4bdc9457SAndroid Build Coastguard Worker $for M in range(1, MR): 71*4bdc9457SAndroid Build Coastguard Worker $for N in range(NR): 72*4bdc9457SAndroid Build Coastguard Worker int32x4_t vacc${M}x${N} = vacc0x${N}; 73*4bdc9457SAndroid Build Coastguard Worker 74*4bdc9457SAndroid Build Coastguard Worker size_t p = ks; 75*4bdc9457SAndroid Build Coastguard Worker do { 76*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 77*4bdc9457SAndroid Build Coastguard Worker const int8_t* restrict a${M} = a[${M}]; 78*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(a${M} != zero) { 79*4bdc9457SAndroid Build Coastguard Worker a${M} = (const int8_t*) ((uintptr_t) a${M} + a_offset); 80*4bdc9457SAndroid Build Coastguard Worker } 81*4bdc9457SAndroid Build Coastguard Worker a += ${MR}; 82*4bdc9457SAndroid Build Coastguard Worker 83*4bdc9457SAndroid Build Coastguard Worker size_t k = kc; 84*4bdc9457SAndroid Build Coastguard Worker $if MLA: 85*4bdc9457SAndroid Build Coastguard Worker // 2x partial unrolled loop to load 16 bytes at a time using MLA. 86*4bdc9457SAndroid Build Coastguard Worker while (k >= 16 * sizeof(int8_t)) { 87*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 88*4bdc9457SAndroid Build Coastguard Worker const int8x8_t va${M}x0 = vld1_s8(a${M}); a${M} += 8; 89*4bdc9457SAndroid Build Coastguard Worker const int8x8_t va${M}x1 = vld1_s8(a${M}); a${M} += 8; 90*4bdc9457SAndroid Build Coastguard Worker 91*4bdc9457SAndroid Build Coastguard Worker $for N in range(NR): 92*4bdc9457SAndroid Build Coastguard Worker const int8x8_t vb${N}x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof( int8_t)); 93*4bdc9457SAndroid Build Coastguard Worker 94*4bdc9457SAndroid Build Coastguard Worker $for N in range(NR): 95*4bdc9457SAndroid Build Coastguard Worker const int8x8_t vb${N}x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof( int8_t)); 96*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 97*4bdc9457SAndroid Build Coastguard Worker int16x8_t vprod${M}x${N} = vmull_s8(vb${N}x0, va${M}x0); 98*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 99*4bdc9457SAndroid Build Coastguard Worker vprod${M}x${N} = vmlal_s8(vprod${M}x${N}, vb${N}x1, va${M}x1); 100*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 101*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${N} = vpadalq_s16(vacc${M}x${N}, vprod${M}x${N}); 102*4bdc9457SAndroid Build Coastguard Worker 103*4bdc9457SAndroid Build Coastguard Worker k -= 16 * sizeof(int8_t); 104*4bdc9457SAndroid Build Coastguard Worker } 105*4bdc9457SAndroid Build Coastguard Worker 106*4bdc9457SAndroid Build Coastguard Worker // Handle 8 bytes at a time using MUL. 107*4bdc9457SAndroid Build Coastguard Worker ${"if" if MLA else "while"} (k != 0) { 108*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 109*4bdc9457SAndroid Build Coastguard Worker const int8x8_t va${M} = vld1_s8(a${M}); a${M} += 8; 110*4bdc9457SAndroid Build Coastguard Worker 111*4bdc9457SAndroid Build Coastguard Worker $for N in range(NR): 112*4bdc9457SAndroid Build Coastguard Worker const int8x8_t vb${N} = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t)); 113*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 114*4bdc9457SAndroid Build Coastguard Worker const int16x8_t vprod${M}x${N} = vmull_s8(vb${N}, va${M}); 115*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 116*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${N} = vpadalq_s16(vacc${M}x${N}, vprod${M}x${N}); 117*4bdc9457SAndroid Build Coastguard Worker 118*4bdc9457SAndroid Build Coastguard Worker k -= 8 * sizeof(int8_t); 119*4bdc9457SAndroid Build Coastguard Worker } 120*4bdc9457SAndroid Build Coastguard Worker 121*4bdc9457SAndroid Build Coastguard Worker p -= ${MR} * sizeof(void*); 122*4bdc9457SAndroid Build Coastguard Worker } while (p != 0); 123*4bdc9457SAndroid Build Coastguard Worker 124*4bdc9457SAndroid Build Coastguard Worker#if XNN_ARCH_ARM64 125*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 126*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 127*4bdc9457SAndroid Build Coastguard Worker const int32x4_t vsum${M}x${ABC[N:N+2]} = vpaddq_s32(vacc${M}x${N}, vacc${M}x${N+1}); 128*4bdc9457SAndroid Build Coastguard Worker const int32x4_t vsum${M}x${ABC[N+2:N+4]} = vpaddq_s32(vacc${M}x${N+2}, vacc${M}x${N+3}); 129*4bdc9457SAndroid Build Coastguard Worker 130*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 131*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 132*4bdc9457SAndroid Build Coastguard Worker int32x4_t vacc${M}x${ABC[N:N+4]} = vpaddq_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]}); 133*4bdc9457SAndroid Build Coastguard Worker#else 134*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 135*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 136*4bdc9457SAndroid Build Coastguard Worker const int32x2_t vpsum${M}x${ABC[N]} = vadd_s32(vget_low_s32(vacc${M}x${N}), vget_high_s32(vacc${M}x${N})); 137*4bdc9457SAndroid Build Coastguard Worker const int32x2_t vpsum${M}x${ABC[N+1]} = vadd_s32(vget_low_s32(vacc${M}x${N+1}), vget_high_s32(vacc${M}x${N+1})); 138*4bdc9457SAndroid Build Coastguard Worker const int32x2_t vpsum${M}x${ABC[N+2]} = vadd_s32(vget_low_s32(vacc${M}x${N+2}), vget_high_s32(vacc${M}x${N+2})); 139*4bdc9457SAndroid Build Coastguard Worker const int32x2_t vpsum${M}x${ABC[N+3]} = vadd_s32(vget_low_s32(vacc${M}x${N+3}), vget_high_s32(vacc${M}x${N+3})); 140*4bdc9457SAndroid Build Coastguard Worker const int32x2_t vsum${M}x${ABC[N:N+2]} = vpadd_s32(vpsum${M}x${ABC[N]}, vpsum${M}x${ABC[N+1]}); 141*4bdc9457SAndroid Build Coastguard Worker const int32x2_t vsum${M}x${ABC[N+2:N+4]} = vpadd_s32(vpsum${M}x${ABC[N+2]}, vpsum${M}x${ABC[N+3]}); 142*4bdc9457SAndroid Build Coastguard Worker int32x4_t vacc${M}x${ABC[N:N+4]} = vcombine_s32(vsum${M}x${ABC[N:N+2]}, vsum${M}x${ABC[N+2:N+4]} ); 143*4bdc9457SAndroid Build Coastguard Worker#endif 144*4bdc9457SAndroid Build Coastguard Worker 145*4bdc9457SAndroid Build Coastguard Worker $if REQUANTIZATION == "RNDNU": 146*4bdc9457SAndroid Build Coastguard Worker const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_pre_shift); 147*4bdc9457SAndroid Build Coastguard Worker const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.multiplier); 148*4bdc9457SAndroid Build Coastguard Worker const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.right_post_shift); 149*4bdc9457SAndroid Build Coastguard Worker 150*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 151*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 152*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+4]} = vqshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_pre_shift); 153*4bdc9457SAndroid Build Coastguard Worker 154*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 155*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 156*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+4]} = vqdmulhq_s32(vacc${M}x${ABC[N:N+4]}, vmultiplier); 157*4bdc9457SAndroid Build Coastguard Worker 158*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 159*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 160*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+4]} = vrshlq_s32(vacc${M}x${ABC[N:N+4]}, vright_post_shift); 161*4bdc9457SAndroid Build Coastguard Worker $elif REQUANTIZATION == "FP32": 162*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 163*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 164*4bdc9457SAndroid Build Coastguard Worker float32x4_t vfpacc${M}x${ABC[N:N+4]} = vcvtq_f32_s32(vacc${M}x${ABC[N:N+4]}); 165*4bdc9457SAndroid Build Coastguard Worker 166*4bdc9457SAndroid Build Coastguard Worker $if CHANNELWISE: 167*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 168*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vscale${ABC[N:N+4]} = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4); 169*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 170*4bdc9457SAndroid Build Coastguard Worker vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale${ABC[N:N+4]}); 171*4bdc9457SAndroid Build Coastguard Worker $else: 172*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vscale = vld1q_dup_f32(¶ms->${PARAMS_STRUCT}.scale); 173*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 174*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 175*4bdc9457SAndroid Build Coastguard Worker vfpacc${M}x${ABC[N:N+4]} = vmulq_f32(vfpacc${M}x${ABC[N:N+4]}, vscale); 176*4bdc9457SAndroid Build Coastguard Worker 177*4bdc9457SAndroid Build Coastguard Worker $if ARMV8: 178*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 179*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 180*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+4]} = vcvtnq_s32_f32(vfpacc${M}x${ABC[N:N+4]}); 181*4bdc9457SAndroid Build Coastguard Worker $else: 182*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vmagic_bias = vld1q_dup_f32(¶ms->${PARAMS_STRUCT}.magic_bias); 183*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 184*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 185*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${M}x${ABC[N:N+4]}, vmagic_bias)); 186*4bdc9457SAndroid Build Coastguard Worker 187*4bdc9457SAndroid Build Coastguard Worker const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(¶ms->${PARAMS_STRUCT}.magic_bias_less_output_zero_point); 188*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 189*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 4): 190*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+4]} = vqsubq_s32(vacc${M}x${ABC[N:N+4]}, vmagic_bias_less_output_zero_point); 191*4bdc9457SAndroid Build Coastguard Worker 192*4bdc9457SAndroid Build Coastguard Worker $if REQUANTIZATION != "FP32" or ARMV8: 193*4bdc9457SAndroid Build Coastguard Worker const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->${PARAMS_STRUCT}.output_zero_point); 194*4bdc9457SAndroid Build Coastguard Worker#if XNN_ARCH_ARM64 195*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 196*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 8): 197*4bdc9457SAndroid Build Coastguard Worker int16x8_t vacc${M}x${ABC[N:N+8]} = vqmovn_high_s32(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vacc${M}x${ABC[N+4:N+8]}); 198*4bdc9457SAndroid Build Coastguard Worker 199*4bdc9457SAndroid Build Coastguard Worker $if REQUANTIZATION != "FP32" or ARMV8: 200*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 201*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 8): 202*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vacc${M}x${ABC[N:N+8]}, voutput_zero_point); 203*4bdc9457SAndroid Build Coastguard Worker 204*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 205*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 16): 206*4bdc9457SAndroid Build Coastguard Worker $if N + 8 < NR: 207*4bdc9457SAndroid Build Coastguard Worker int8x16_t vout${M}x${ABC[N:N+16]} = vqmovn_high_s16(vqmovn_s16(vacc${M}x${ABC[N:N+8]}), vacc${M}x${ABC[N+8:N+16]}); 208*4bdc9457SAndroid Build Coastguard Worker $elif M % 2 == 1: 209*4bdc9457SAndroid Build Coastguard Worker int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vqmovn_high_s16(vqmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vacc${M}x${ABC[N:N+8]}); 210*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 211*4bdc9457SAndroid Build Coastguard Worker int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]}); 212*4bdc9457SAndroid Build Coastguard Worker#else 213*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 214*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 8): 215*4bdc9457SAndroid Build Coastguard Worker int16x8_t vacc${M}x${ABC[N:N+8]} = vcombine_s16(vqmovn_s32(vacc${M}x${ABC[N:N+4]}), vqmovn_s32(vacc${M}x${ABC[N+4:N+8]})); 216*4bdc9457SAndroid Build Coastguard Worker 217*4bdc9457SAndroid Build Coastguard Worker $if REQUANTIZATION != "FP32" or ARMV8: 218*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 219*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 8): 220*4bdc9457SAndroid Build Coastguard Worker vacc${M}x${ABC[N:N+8]} = vqaddq_s16(vacc${M}x${ABC[N:N+8]}, voutput_zero_point); 221*4bdc9457SAndroid Build Coastguard Worker 222*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 223*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 16): 224*4bdc9457SAndroid Build Coastguard Worker $if N + 8 < NR: 225*4bdc9457SAndroid Build Coastguard Worker int8x16_t vout${M}x${ABC[N:N+16]} = vcombine_s8(vqmovn_s16(vacc${M}x${ABC[N:N+8]}), vqmovn_s16(vacc${M}x${ABC[N+8:N+16]})); 226*4bdc9457SAndroid Build Coastguard Worker $elif M % 2 == 1: 227*4bdc9457SAndroid Build Coastguard Worker int8x16_t vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vcombine_s8(vqmovn_s16(vacc${M-1}x${ABC[N:N+8]}), vqmovn_s16(vacc${M}x${ABC[N:N+8]})); 228*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 229*4bdc9457SAndroid Build Coastguard Worker int8x8_t vout${M}x${ABC[N:N+8]} = vqmovn_s16(vacc${M}x${ABC[N:N+8]}); 230*4bdc9457SAndroid Build Coastguard Worker#endif 231*4bdc9457SAndroid Build Coastguard Worker 232*4bdc9457SAndroid Build Coastguard Worker $if NR == 8 and MR == 1: 233*4bdc9457SAndroid Build Coastguard Worker const int8x8_t voutput_min = vld1_dup_s8(¶ms->${PARAMS_STRUCT}.output_min); 234*4bdc9457SAndroid Build Coastguard Worker $else: 235*4bdc9457SAndroid Build Coastguard Worker const int8x16_t voutput_min = vld1q_dup_s8(¶ms->${PARAMS_STRUCT}.output_min); 236*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 237*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 16): 238*4bdc9457SAndroid Build Coastguard Worker $if N + 8 < NR: 239*4bdc9457SAndroid Build Coastguard Worker vout${M}x${ABC[N:N+16]} = vmaxq_s8(vout${M}x${ABC[N:N+16]}, voutput_min); 240*4bdc9457SAndroid Build Coastguard Worker $elif M % 2 == 1: 241*4bdc9457SAndroid Build Coastguard Worker vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vmaxq_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_min); 242*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 243*4bdc9457SAndroid Build Coastguard Worker $if NR == 8 and MR == 1: 244*4bdc9457SAndroid Build Coastguard Worker vout${M}x${ABC[N:N+8]} = vmax_s8(vout${M}x${ABC[N:N+8]}, voutput_min); 245*4bdc9457SAndroid Build Coastguard Worker $else: 246*4bdc9457SAndroid Build Coastguard Worker vout${M}x${ABC[N:N+8]} = vmax_s8(vout${M}x${ABC[N:N+8]}, vget_low_s8(voutput_min)); 247*4bdc9457SAndroid Build Coastguard Worker 248*4bdc9457SAndroid Build Coastguard Worker $if NR == 8 and MR == 1: 249*4bdc9457SAndroid Build Coastguard Worker const int8x8_t voutput_max = vld1_dup_s8(¶ms->${PARAMS_STRUCT}.output_max); 250*4bdc9457SAndroid Build Coastguard Worker $else: 251*4bdc9457SAndroid Build Coastguard Worker const int8x16_t voutput_max = vld1q_dup_s8(¶ms->${PARAMS_STRUCT}.output_max); 252*4bdc9457SAndroid Build Coastguard Worker $for M in range(MR): 253*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 16): 254*4bdc9457SAndroid Build Coastguard Worker $if N + 8 < NR: 255*4bdc9457SAndroid Build Coastguard Worker vout${M}x${ABC[N:N+16]} = vminq_s8(vout${M}x${ABC[N:N+16]}, voutput_max); 256*4bdc9457SAndroid Build Coastguard Worker $elif M % 2 == 1: 257*4bdc9457SAndroid Build Coastguard Worker vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]} = vminq_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]}, voutput_max); 258*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 259*4bdc9457SAndroid Build Coastguard Worker $if NR == 8 and MR == 1: 260*4bdc9457SAndroid Build Coastguard Worker vout${M}x${ABC[N:N+8]} = vmin_s8(vout${M}x${ABC[N:N+8]}, voutput_max); 261*4bdc9457SAndroid Build Coastguard Worker $else: 262*4bdc9457SAndroid Build Coastguard Worker vout${M}x${ABC[N:N+8]} = vmin_s8(vout${M}x${ABC[N:N+8]}, vget_low_s8(voutput_max)); 263*4bdc9457SAndroid Build Coastguard Worker 264*4bdc9457SAndroid Build Coastguard Worker if (nc >= ${NR}) { 265*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 266*4bdc9457SAndroid Build Coastguard Worker $for N in range(0, NR, 16): 267*4bdc9457SAndroid Build Coastguard Worker $if N + 8 < NR: 268*4bdc9457SAndroid Build Coastguard Worker vst1q_s8(c${M} + ${N}, vout${M}x${ABC[N:N+16]}); 269*4bdc9457SAndroid Build Coastguard Worker $elif M % 2 == 1: 270*4bdc9457SAndroid Build Coastguard Worker vst1_s8(c${M} + ${N}, vget_high_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 271*4bdc9457SAndroid Build Coastguard Worker vst1_s8(c${M-1} + ${N}, vget_low_s8(vout${M-1}x${ABC[N:N+8]}_${M}x${ABC[N:N+8]})); 272*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 273*4bdc9457SAndroid Build Coastguard Worker vst1_s8(c${M} + ${N}, vout${M}x${ABC[N:N+8]}); 274*4bdc9457SAndroid Build Coastguard Worker 275*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 276*4bdc9457SAndroid Build Coastguard Worker c${M} = (int8_t*) ((uintptr_t) c${M} + cn_stride); 277*4bdc9457SAndroid Build Coastguard Worker 278*4bdc9457SAndroid Build Coastguard Worker a = (const int8_t**restrict) ((uintptr_t) a - ks); 279*4bdc9457SAndroid Build Coastguard Worker 280*4bdc9457SAndroid Build Coastguard Worker nc -= ${NR}; 281*4bdc9457SAndroid Build Coastguard Worker } else { 282*4bdc9457SAndroid Build Coastguard Worker $if NR == 16: 283*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 284*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 285*4bdc9457SAndroid Build Coastguard Worker int8x16_t vout${M-1}x01234567_${M}x01234567 = vcombine_s8(vget_low_s8(vout${M-1}x0123456789ABCDEF), vget_low_s8(vout${M}x0123456789ABCDEF)); 286*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 287*4bdc9457SAndroid Build Coastguard Worker int8x8_t vout${M}x01234567 = vget_low_s8(vout${M}x0123456789ABCDEF); 288*4bdc9457SAndroid Build Coastguard Worker if (nc & 8) { 289*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 290*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 291*4bdc9457SAndroid Build Coastguard Worker vst1_s8(c${M}, vget_high_s8(vout${M-1}x01234567_${M}x01234567)); c${M} += 8; 292*4bdc9457SAndroid Build Coastguard Worker vst1_s8(c${M-1}, vget_low_s8(vout${M-1}x01234567_${M}x01234567)); c${M-1} += 8; 293*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 294*4bdc9457SAndroid Build Coastguard Worker vst1_s8(c${M}, vout${M}x01234567); c${M} += 8; 295*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 296*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 297*4bdc9457SAndroid Build Coastguard Worker vout${M-1}x01234567_${M}x01234567 = vcombine_s8(vget_high_s8(vout${M-1}x0123456789ABCDEF), vget_high_s8(vout${M}x0123456789ABCDEF)); 298*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 299*4bdc9457SAndroid Build Coastguard Worker vout${M}x01234567 = vget_high_s8(vout${M}x0123456789ABCDEF); 300*4bdc9457SAndroid Build Coastguard Worker } 301*4bdc9457SAndroid Build Coastguard Worker if (nc & 4) { 302*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 303*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 304*4bdc9457SAndroid Build Coastguard Worker vst1q_lane_u32((void*) c${M}, vreinterpretq_u32_s8(vout${M-1}x01234567_${M}x01234567), 2); c${M} += 4; 305*4bdc9457SAndroid Build Coastguard Worker vst1q_lane_u32((void*) c${M-1}, vreinterpretq_u32_s8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 4; 306*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 307*4bdc9457SAndroid Build Coastguard Worker vst1_lane_u32((void*) c${M}, vreinterpret_u32_s8(vout${M}x01234567), 0); c${M} += 4; 308*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 309*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 310*4bdc9457SAndroid Build Coastguard Worker vout${M-1}x01234567_${M}x01234567 = vextq_s8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 4); 311*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 312*4bdc9457SAndroid Build Coastguard Worker vout${M}x01234567 = vext_s8(vout${M}x01234567, vout${M}x01234567, 4); 313*4bdc9457SAndroid Build Coastguard Worker } 314*4bdc9457SAndroid Build Coastguard Worker if (nc & 2) { 315*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 316*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 317*4bdc9457SAndroid Build Coastguard Worker vst1q_lane_u16((void*) c${M}, vreinterpretq_u16_s8(vout${M-1}x01234567_${M}x01234567), 4); c${M} += 2; 318*4bdc9457SAndroid Build Coastguard Worker vst1q_lane_u16((void*) c${M-1}, vreinterpretq_u16_s8(vout${M-1}x01234567_${M}x01234567), 0); c${M-1} += 2; 319*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 320*4bdc9457SAndroid Build Coastguard Worker vst1_lane_u16((void*) c${M}, vreinterpret_u16_s8(vout${M}x01234567), 0); c${M} += 2; 321*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 322*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 323*4bdc9457SAndroid Build Coastguard Worker vout${M-1}x01234567_${M}x01234567 = vextq_s8(vout${M-1}x01234567_${M}x01234567, vout${M-1}x01234567_${M}x01234567, 2); 324*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 325*4bdc9457SAndroid Build Coastguard Worker vout${M}x01234567 = vext_s8(vout${M}x01234567, vout${M}x01234567, 2); 326*4bdc9457SAndroid Build Coastguard Worker } 327*4bdc9457SAndroid Build Coastguard Worker if (nc & 1) { 328*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(MR)): 329*4bdc9457SAndroid Build Coastguard Worker $if M % 2 == 1: 330*4bdc9457SAndroid Build Coastguard Worker vst1q_lane_s8(c${M}, vout${M-1}x01234567_${M}x01234567, 8); 331*4bdc9457SAndroid Build Coastguard Worker vst1q_lane_s8(c${M-1}, vout${M-1}x01234567_${M}x01234567, 0); 332*4bdc9457SAndroid Build Coastguard Worker $elif M + 1 == MR: 333*4bdc9457SAndroid Build Coastguard Worker vst1_lane_s8(c${M}, vout${M}x01234567, 0); 334*4bdc9457SAndroid Build Coastguard Worker } 335*4bdc9457SAndroid Build Coastguard Worker 336*4bdc9457SAndroid Build Coastguard Worker nc = 0; 337*4bdc9457SAndroid Build Coastguard Worker } 338*4bdc9457SAndroid Build Coastguard Worker } while (nc != 0); 339*4bdc9457SAndroid Build Coastguard Worker} 340