1// Copyright 2020 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert REQUANTIZATION == "FP32" 7$assert DATATYPE in ["QC8", "QS8", "QU8"] 8$assert VARIANT in ["LD128", "EXTENDED"] 9$assert MR <= 4 10#include <assert.h> 11 12#include <immintrin.h> 13 14#include <xnnpack/gemm.h> 15#include <xnnpack/intrinsics-polyfill.h> 16#include <xnnpack/math.h> 17#include <xnnpack/unaligned.h> 18 19 20$GEMM_SUFFIX = "_xw" if VARIANT == "EXTENDED" else "" 21$PARAMS_STRUCT = REQUANTIZATION.lower() + "_avx2" 22$PARAMS_UNION = "xnn_%s_conv_minmax_params" % DATATYPE.lower() 23$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t" 24void xnn_${DATATYPE.lower()}_gemm${GEMM_SUFFIX}_minmax_fp32_ukernel_${MR}x8c8__avx2( 25 size_t mr, 26 size_t nc, 27 size_t kc, 28 const ${XINT8_T}* restrict a, 29 size_t a_stride, 30 const void* restrict w, 31 ${XINT8_T}* restrict c, 32 size_t cm_stride, 33 size_t cn_stride, 34 const union ${PARAMS_UNION} params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 35{ 36 assert(mr != 0); 37 assert(mr <= ${MR}); 38 assert(nc != 0); 39 assert(kc != 0); 40 assert(kc % sizeof(${XINT8_T}) == 0); 41 assert(a != NULL); 42 assert(w != NULL); 43 assert(c != NULL); 44 45 kc = round_up_po2(kc, 8); 46 const ${XINT8_T}* a0 = a; 47 ${XINT8_T}* c0 = c; 48 $for M in range(1, MR): 49 const ${XINT8_T}* a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M-1} + a_stride); 50 ${XINT8_T}* c${M} = (${XINT8_T}*) ((uintptr_t) c${M-1} + cm_stride); 51 $if M % 2 == 0: 52 if XNN_UNPREDICTABLE(mr <= ${M}) { 53 a${M} = a${M-1}; 54 c${M} = c${M-1}; 55 } 56 $elif M + 1 == MR: 57 if XNN_UNPREDICTABLE(mr != ${M+1}) { 58 a${M} = a${M-1}; 59 c${M} = c${M-1}; 60 } 61 $else: 62 if XNN_UNPREDICTABLE(mr < ${M+1}) { 63 a${M} = a${M-1}; 64 c${M} = c${M-1}; 65 } 66 67 do { 68 $for N in range(0, 8, 2): 69 const __m128i vbias0x${N} = _mm_cvtsi32_si128(((const int*) w)[${N}]); 70 const __m128i vbias0x${N+1} = _mm_cvtsi32_si128(((const int*) w)[${N+1}]); 71 __m256i vacc0x${N}${N+1} = _mm256_inserti128_si256(_mm256_castsi128_si256(vbias0x${N}), vbias0x${N+1}, 1); 72 $for M in range(1, MR): 73 $for N in range(0, 8, 2): 74 __m256i vacc${M}x${N}${N+1} = vacc0x${N}${N+1}; 75 w = (const int32_t*) w + 8; 76 77 size_t k = 0; 78 $if DATATYPE == "QU8": 79 const __m256i vb_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.kernel_zero_point); 80 while (k < kc) { 81 $for M in range(MR): 82 const __m128i va${M} = _mm_broadcastq_epi64(_mm_loadl_epi64((const __m128i*) a${M})); 83 $if DATATYPE == "QU8": 84 const __m256i vxa${M} = _mm256_cvtepu8_epi16(va${M}); 85 $else: 86 const __m256i vxa${M} = _mm256_cvtepi8_epi16(va${M}); 87 a${M} += 8; 88 89 $for N in range(0, 8, 2): 90 $if VARIANT == "EXTENDED": 91 $if N == 0: 92 const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) w); 93 $else: 94 const __m256i vxb${N}${N+1} = _mm256_load_si256((const __m256i*) ((const int16_t*) w + ${N * 8})); 95 $else: 96 $if N == 0: 97 const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w); 98 $else: 99 const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const ${XINT8_T}*) w + ${N * 8})); 100 $if DATATYPE == "QU8": 101 const __m256i vxb${N}${N+1} = _mm256_sub_epi16(_mm256_cvtepu8_epi16(vb${N}${N+1}), vb_zero_point); 102 $else: 103 const __m256i vxb${N}${N+1} = _mm256_cvtepi8_epi16(vb${N}${N+1}); 104 105 $for M in range(MR): 106 vacc${M}x${N}${N+1} = _mm256_add_epi32(vacc${M}x${N}${N+1}, _mm256_madd_epi16(vxa${M}, vxb${N}${N+1})); 107 108 $if VARIANT == "EXTENDED": 109 w = (const void*) ((const int16_t*) w + 64); 110 $else: 111 w = (const void*) ((const ${XINT8_T}*) w + 64); 112 k += 8 * sizeof(${XINT8_T}); 113 } 114 115 $for M in range(MR): 116 const __m256i vacc${M}x0213 = _mm256_hadd_epi32(vacc${M}x01, vacc${M}x23); 117 const __m256i vacc${M}x4657 = _mm256_hadd_epi32(vacc${M}x45, vacc${M}x67); 118 119 $for M in range(MR): 120 const __m256i vacc${M}x02461357 = _mm256_hadd_epi32(vacc${M}x0213, vacc${M}x4657); 121 122 const __m256i vpermute_mask = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0); 123 $for M in range(MR): 124 __m256i vacc${M}x01234567 = _mm256_permutevar8x32_epi32(vacc${M}x02461357, vpermute_mask); 125 126 $for M in range(MR): 127 __m256 vscaled${M}x01234567 = _mm256_cvtepi32_ps(vacc${M}x01234567); 128 129 $if DATATYPE == "QC8": 130 const __m256 vscale01234567 = _mm256_load_ps(w); 131 w = (const void*) ((const float*) w + 8); 132 $for M in range(MR): 133 vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale01234567); 134 $else: 135 const __m256 vscale = _mm256_load_ps(params->fp32_avx2.scale); 136 $for M in range(MR): 137 vscaled${M}x01234567 = _mm256_mul_ps(vscaled${M}x01234567, vscale); 138 139 const __m256 voutput_max_less_zero_point = _mm256_load_ps(params->${PARAMS_STRUCT}.output_max_less_zero_point); 140 $for M in range(MR): 141 vscaled${M}x01234567 = _mm256_min_ps(vscaled${M}x01234567, voutput_max_less_zero_point); 142 143 $for M in range(MR): 144 vacc${M}x01234567 = _mm256_cvtps_epi32(vscaled${M}x01234567); 145 146 const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_zero_point); 147 $for M in range(0, MR, 2): 148 __m256i vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_adds_epi16(_mm256_packs_epi32(vacc${M}x01234567, vacc${min(M+1, MR-1)}x01234567), voutput_zero_point); 149 150 $for M in range(0, MR, 2): 151 vacc${M}${min(M+1, MR-1)}x01234567 = _mm256_permute4x64_epi64(vacc${M}${min(M+1, MR-1)}x01234567, _MM_SHUFFLE(3, 1, 2, 0)); 152 153 $if DATATYPE == "QU8": 154 $if MR > 2: 155 __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567); 156 $else: 157 __m256i vout = _mm256_packus_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567); 158 159 vout = _mm256_max_epu8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min)); 160 $else: 161 $if MR > 2: 162 __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc${min(2, MR-1)}${min(3, MR-1)}x01234567); 163 $else: 164 __m256i vout = _mm256_packs_epi16(vacc0${min(1, MR-1)}x01234567, vacc0${min(1, MR-1)}x01234567); 165 166 vout = _mm256_max_epi8(vout, _mm256_load_si256((const __m256i*) params->${PARAMS_STRUCT}.output_min)); 167 168 __m128i vout_lo = _mm256_castsi256_si128(vout); 169 __m128i vout_hi = _mm256_extracti128_si256(vout, 1); 170 171 if (nc >= 8) { 172 _mm_storel_epi64((__m128i*) c0, vout_lo); 173 $if MR > 1: 174 _mm_storel_epi64((__m128i*) c1, vout_hi); 175 $if MR > 2: 176 _mm_storeh_pi((__m64*) c2, _mm_castsi128_ps(vout_lo)); 177 $if MR > 3: 178 _mm_storeh_pi((__m64*) c3, _mm_castsi128_ps(vout_hi)); 179 180 $for M in range(MR): 181 c${M} = (${XINT8_T}*) ((uintptr_t) c${M} + cn_stride); 182 183 $for M in range(MR): 184 a${M} = (const ${XINT8_T}*) ((uintptr_t) a${M} - kc); 185 186 nc -= 8; 187 } else { 188 if (nc & 4) { 189 _mm_storeu_si32(c0, vout_lo); 190 $if MR > 1: 191 _mm_storeu_si32(c1, vout_hi); 192 $if MR > 2: 193 unaligned_store_u32(c2, (uint32_t) _mm_extract_epi32(vout_lo, 2)); 194 $if MR > 3: 195 unaligned_store_u32(c3, (uint32_t) _mm_extract_epi32(vout_hi, 2)); 196 197 $for M in range(MR): 198 c${M} += 4; 199 200 vout_lo = _mm_srli_epi64(vout_lo, 32); 201 vout_hi = _mm_srli_epi64(vout_hi, 32); 202 } 203 if (nc & 2) { 204 unaligned_store_u16(c0, (uint16_t) _mm_extract_epi16(vout_lo, 0)); 205 $if MR > 1: 206 unaligned_store_u16(c1, (uint16_t) _mm_extract_epi16(vout_hi, 0)); 207 $if MR > 2: 208 unaligned_store_u16(c2, (uint16_t) _mm_extract_epi16(vout_lo, 4)); 209 $if MR > 3: 210 unaligned_store_u16(c3, (uint16_t) _mm_extract_epi16(vout_hi, 4)); 211 212 $for M in range(MR): 213 c${M} += 2; 214 215 vout_lo = _mm_srli_epi32(vout_lo, 16); 216 vout_hi = _mm_srli_epi32(vout_hi, 16); 217 } 218 if (nc & 1) { 219 *c0 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 0); 220 $if MR > 1: 221 *c1 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 0); 222 $if MR > 2: 223 *c2 = (${XINT8_T}) _mm_extract_epi8(vout_lo, 8); 224 $if MR > 3: 225 *c3 = (${XINT8_T}) _mm_extract_epi8(vout_hi, 8); 226 } 227 228 nc = 0; 229 } 230 } while (nc != 0); 231} 232