1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert REQUANTIZATION == "FP32" 7$assert DATATYPE in ["QC8", "QS8", "QU8"] 8$assert VARIANT in ["LD64", "LD128"] 9$assert MR <= 4 10#include <assert.h> 11 12#include <wasm_simd128.h> 13 14#include <xnnpack/igemm.h> 15#include <xnnpack/math.h> 16 17 18$PARAMS_STRUCT = REQUANTIZATION.lower() + "_wasmsimd" 19$PARAMS_UNION = "xnn_%s_conv_minmax_params" % DATATYPE.lower() 20$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t" 21$WASM_X16X8_LOAD8X8 = "wasm_u16x8_load8x8" if DATATYPE == "QU8" else "wasm_i16x8_load8x8" 22$WASM_X8X16_NARROW_I16X8 = "wasm_u8x16_narrow_i16x8" if DATATYPE == "QU8" else "wasm_i8x16_narrow_i16x8" 23$WASM_X8X16_MIN = "wasm_u8x16_min" if DATATYPE == "QU8" else "wasm_i8x16_min" 24void xnn_${DATATYPE.lower()}_igemm_minmax_fp32_ukernel_${MR}x4c2s4__wasmsimd_dot16x2_${VARIANT.lower()}( 25 size_t mr, 26 size_t nc, 27 size_t kc, 28 size_t ks, 29 const ${XINT8_T}** restrict a, 30 const void* restrict w, 31 ${XINT8_T}* restrict c, 32 size_t cm_stride, 33 size_t cn_stride, 34 size_t a_offset, 35 const ${XINT8_T}* zero, 36 const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 37{ 38 assert(mr != 0); 39 assert(mr <= ${MR}); 40 assert(nc != 0); 41 assert(kc != 0); 42 assert(ks != 0); 43 assert(ks % (${MR} * sizeof(void*)) == 0); 44 assert(a_offset % sizeof(${XINT8_T}) == 0); 45 assert(a != NULL); 46 assert(w != NULL); 47 assert(c != NULL); 48 49 ${XINT8_T}* c0 = c; 50 $for M in range(1, MR): 51 ${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride); 52 $if M % 2 == 0: 53 if XNN_UNPREDICTABLE(mr <= ${M}) { 54 c${M} = c${M-1}; 55 } 56 $elif M + 1 == MR: 57 if XNN_UNPREDICTABLE(mr != ${M+1}) { 58 c${M} = c${M-1}; 59 } 60 $else: 61 if XNN_UNPREDICTABLE(mr < ${M+1}) { 62 c${M} = c${M-1}; 63 } 64 65 kc = round_up_po2(kc, 8 * sizeof(${XINT8_T})); 66 do { 67 v128_t vacc0x0123 = wasm_v128_load(w); 68 $for M in range(1, MR): 69 v128_t vacc${M}x0123 = vacc0x0123; 70 w = (const void*) ((const int32_t*) w + 4); 71 72 size_t p = ks; 73 do { 74 $for M in range(MR): 75 const ${XINT8_T}* restrict a${M} = a[${M}]; 76 if XNN_UNPREDICTABLE(a${M} != zero) { 77 a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} + a_offset); 78 } 79 a += ${MR}; 80 81 $if DATATYPE == "QU8": 82 const v128_t vb_zero_point = wasm_v128_load64_splat(params->${PARAMS_STRUCT}.kernel_zero_point); 83 size_t k = kc; 84 do { 85 $for M in range(MR): 86 v128_t vxa${M} = ${WASM_X16X8_LOAD8X8}(a${M}); 87 a${M} += 8; 88 89 $if VARIANT == "LD128": 90 $for K in range(0, 4, 2): 91 $if K == 0: 92 const v128_t vb${K}${K+1} = wasm_v128_load(w); 93 $else: 94 const v128_t vb${K}${K+1} = wasm_v128_load((const ${XINT8_T}*) w + ${K * 8}); 95 $if DATATYPE == "QU8": 96 const v128_t vxb${K} = wasm_i16x8_sub(wasm_u16x8_extend_low_u8x16(vb${K}${K+1}), vb_zero_point); 97 const v128_t vxb${K+1} = wasm_i16x8_sub(wasm_u16x8_extend_high_u8x16(vb${K}${K+1}), vb_zero_point); 98 $else: 99 const v128_t vxb${K} = wasm_i16x8_extend_low_i8x16(vb${K}${K+1}); 100 const v128_t vxb${K+1} = wasm_i16x8_extend_high_i8x16(vb${K}${K+1}); 101 102 $for M in range(MR): 103 vacc${M}x0123 = wasm_i32x4_add(vacc${M}x0123, wasm_i32x4_dot_i16x8(vxa${M}, vxb${K})); 104 vxa${M} = wasm_v32x4_shuffle(vxa${M}, vxa${M}, 1, 2, 3, 4); 105 106 $for M in range(MR): 107 vacc${M}x0123 = wasm_i32x4_add(vacc${M}x0123, wasm_i32x4_dot_i16x8(vxa${M}, vxb${K+1})); 108 $if K + 2 != 4: 109 vxa${M} = wasm_v32x4_shuffle(vxa${M}, vxa${M}, 1, 2, 3, 4); 110 $else: 111 $for K in range(4): 112 $if K == 0: 113 $if DATATYPE == "QU8": 114 const v128_t vxb${K} = wasm_i16x8_sub(wasm_u16x8_load8x8(w), vb_zero_point); 115 $else: 116 const v128_t vxb${K} = wasm_i16x8_load8x8(w); 117 $else: 118 $if DATATYPE == "QU8": 119 const v128_t vxb${K} = wasm_i16x8_sub(wasm_u16x8_load8x8((const ${XINT8_T}*) w + ${K * 8}), vb_zero_point); 120 $else: 121 const v128_t vxb${K} = wasm_i16x8_load8x8((const ${XINT8_T}*) w + ${K * 8}); 122 123 $for M in range(MR): 124 vacc${M}x0123 = wasm_i32x4_add(vacc${M}x0123, wasm_i32x4_dot_i16x8(vxa${M}, vxb${K})); 125 $if K + 1 != 4: 126 vxa${M} = wasm_v32x4_shuffle(vxa${M}, vxa${M}, 1, 2, 3, 4); 127 128 w = (const ${XINT8_T}*) w + 32; 129 k -= 8 * sizeof(${XINT8_T}); 130 } while (k != 0); 131 p -= ${MR} * sizeof(void*); 132 } while (p != 0); 133 134 $for M in range(MR): 135 vacc${M}x0123 = wasm_f32x4_convert_i32x4(vacc${M}x0123); 136 137 $if DATATYPE == "QC8": 138 const v128_t vscale0123 = wasm_v128_load(w); 139 w = (const void*) ((const float*) w + 4); 140 $for M in range(MR): 141 vacc${M}x0123 = wasm_f32x4_mul(vacc${M}x0123, vscale0123); 142 $else: 143 const v128_t vscale = wasm_v128_load64_splat(params->${PARAMS_STRUCT}.scale); 144 $for M in range(MR): 145 vacc${M}x0123 = wasm_f32x4_mul(vacc${M}x0123, vscale); 146 147 const v128_t vmagic_bias = wasm_v128_load64_splat(params->${PARAMS_STRUCT}.magic_bias); 148 $for M in range(MR): 149 vacc${M}x0123 = wasm_f32x4_add(vacc${M}x0123, vmagic_bias); 150 151 const v128_t vmagic_min = wasm_v128_load64_splat(params->${PARAMS_STRUCT}.magic_min); 152 $for M in range(MR): 153 vacc${M}x0123 = wasm_i32x4_max(vacc${M}x0123, vmagic_min); 154 155 const v128_t vmagic_bias_less_output_zero_point = wasm_v128_load64_splat(params->${PARAMS_STRUCT}.magic_bias_less_output_zero_point); 156 $for M in range(MR): 157 vacc${M}x0123 = wasm_i32x4_sub(vacc${M}x0123, vmagic_bias_less_output_zero_point); 158 159 $for M in range(0, MR, 2): 160 v128_t vacc${M}${min(M+1, MR-1)}x0123 = wasm_i16x8_narrow_i32x4(vacc${M}x0123, vacc${min(M+1, MR-1)}x0123); 161 162 $if MR > 2: 163 v128_t vout = ${WASM_X8X16_NARROW_I16X8}(vacc0${min(1, MR-1)}x0123, vacc${min(2, MR-1)}${min(3, MR-1)}x0123); 164 $else: 165 v128_t vout = ${WASM_X8X16_NARROW_I16X8}(vacc0${min(1, MR-1)}x0123, vacc0${min(1, MR-1)}x0123); 166 167 const v128_t voutput_max = wasm_v128_load64_splat(params->${PARAMS_STRUCT}.output_max); 168 vout = ${WASM_X8X16_MIN}(vout, voutput_max); 169 170 if (nc >= 4) { 171 $for M in reversed(range(MR)): 172 *((float*) c${M}) = (float) wasm_f32x4_extract_lane(vout, ${M}); 173 174 $for M in reversed(range(MR)): 175 c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride); 176 177 a = (const ${XINT8_T}**restrict) ((uintptr_t) a - ks); 178 179 nc -= 4; 180 } else { 181 $for M in reversed(range(MR)): 182 uint32_t vout${M} = wasm_i32x4_extract_lane(vout, ${M}); 183 if (nc & 2) { 184 $for M in reversed(range(MR)): 185 *((uint16_t*) c${M}) = (uint16_t) vout${M}; 186 vout${M} >>= 16; 187 c${M} += 2; 188 } 189 if (nc & 1) { 190 $for M in reversed(range(MR)): 191 *c${M} = (${XINT8_T}) vout${M}; 192 } 193 194 nc = 0; 195 } 196 } while (nc != 0); 197} 198