xref: /aosp_15_r20/external/XNNPACK/src/qs8-vadd/avx512skx-mul32-ld128.c.in (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1// Copyright 2021 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert DATATYPE in ["QS8", "QU8"]
7$assert BATCH_TILE % 16 == 0
8$assert BATCH_TILE >= 16
9$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
10#include <assert.h>
11
12#include <immintrin.h>
13
14#include <xnnpack/intrinsics-polyfill.h>
15#include <xnnpack/vadd.h>
16
17
18$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE]
19$_MM512_CVTEPX8_EPI32 = {"QS8": "_mm512_cvtepi8_epi32", "QU8": "_mm512_cvtepu8_epi32"}[DATATYPE]
20$_MM256_PACKXS_EPI16 = {"QS8": "_mm256_packs_epi16", "QU8": "_mm256_packus_epi16"}[DATATYPE]
21$_MM_PACKXS_EPI16 = {"QS8": "_mm_packs_epi16", "QU8": "_mm_packus_epi16"}[DATATYPE]
22$_MM256_MIN_EPX8 = {"QS8": "_mm256_min_epi8", "QU8": "_mm256_min_epu8"}[DATATYPE]
23$_MM256_MAX_EPX8 = {"QS8": "_mm256_max_epi8", "QU8": "_mm256_max_epu8"}[DATATYPE]
24$_MM_MIN_EPX8 = {"QS8": "_mm_min_epi8", "QU8": "_mm_min_epu8"}[DATATYPE]
25$_MM_MAX_EPX8 = {"QS8": "_mm_max_epi8", "QU8": "_mm_max_epu8"}[DATATYPE]
26void xnn_${DATATYPE.lower()}_vadd_minmax_ukernel__avx512skx_mul32_ld128_x${BATCH_TILE}(
27    size_t n,
28    const ${XINT8_T}* input_a,
29    const ${XINT8_T}* input_b,
30    ${XINT8_T}* output,
31    const union xnn_${DATATYPE.lower()}_add_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
32{
33  const __m512i vbias = _mm512_load_si512(params->avx512.bias);
34  const __m512i va_multiplier = _mm512_load_si512(params->avx512.a_multiplier);
35  const __m512i vb_multiplier = _mm512_load_si512(params->avx512.b_multiplier);
36  const __m128i vshift = _mm_load_si128((const __m128i*) params->avx512.shift);
37  $if BATCH_TILE > 16:
38    const __m512i voutput_zero_point = _mm512_load_si512(params->avx512.output_zero_point);
39    const __m256i voutput_min = _mm256_load_si256((const __m256i*) params->avx512.output_min);
40    const __m256i voutput_max = _mm256_load_si256((const __m256i*) params->avx512.output_max);
41  $else:
42    const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx512.output_zero_point);
43    const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx512.output_min);
44    const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx512.output_max);
45
46  for (; n >= ${BATCH_TILE} * sizeof(${XINT8_T}); n -= ${BATCH_TILE} * sizeof(${XINT8_T})) {
47    const __m512i va${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) input_a));
48    const __m512i vb${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) input_b));
49    $for N in range(16, BATCH_TILE, 16):
50      const __m512i va${ABC[N:N+16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) (input_a + ${N})));
51      const __m512i vb${ABC[N:N+16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) (input_b + ${N})));
52    input_a += ${BATCH_TILE};
53    input_b += ${BATCH_TILE};
54
55    $for N in range(0, BATCH_TILE, 16):
56      __m512i vacc${ABC[N:N+16]} = _mm512_add_epi32(vbias, _mm512_mullo_epi32(va${ABC[N:N+16]}, va_multiplier));
57
58    $for N in range(0, BATCH_TILE, 16):
59      vacc${ABC[N:N+16]} = _mm512_add_epi32(vacc${ABC[N:N+16]}, _mm512_mullo_epi32(vb${ABC[N:N+16]}, vb_multiplier));
60
61    $for N in range(0, BATCH_TILE, 16):
62      vacc${ABC[N:N+16]} = _mm512_sra_epi32(vacc${ABC[N:N+16]}, vshift);
63
64    $for N in range(0, BATCH_TILE, 32):
65      $if N + 16 < BATCH_TILE:
66        __m512i vout${ABC[N:N+4]}${ABC[N+16:N+20]}${ABC[N+4:N+8]}${ABC[N+20:N+24]}${ABC[N+8:N+12]}${ABC[N+24:N+28]}${ABC[N+12:N+16]}${ABC[N+28:N+32]} = _mm512_adds_epi16(_mm512_packs_epi32(vacc${ABC[N:N+16]}, vacc${ABC[N+16:N+32]}), voutput_zero_point);
67      $elif BATCH_TILE > 16:
68        __m256i vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[N:N+16]}), _mm512_extracti32x8_epi32(vacc${ABC[N:N+16]}, 1)), _mm512_castsi512_si256(voutput_zero_point));
69      $else:
70        __m256i vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[N:N+16]}), _mm512_extracti32x8_epi32(vacc${ABC[N:N+16]}, 1)), voutput_zero_point);
71
72    $for N in range(0, BATCH_TILE, 32):
73      $if N + 16 < BATCH_TILE:
74        __m256i vout${ABC[N:N+32]} = _mm256_permutevar8x32_epi32(${_MM256_PACKXS_EPI16}(_mm512_castsi512_si256(vout${ABC[N:N+4]}${ABC[N+16:N+20]}${ABC[N+4:N+8]}${ABC[N+20:N+24]}${ABC[N+8:N+12]}${ABC[N+24:N+28]}${ABC[N+12:N+16]}${ABC[N+28:N+32]}), _mm512_extracti32x8_epi32(vout${ABC[N:N+4]}${ABC[N+16:N+20]}${ABC[N+4:N+8]}${ABC[N+20:N+24]}${ABC[N+8:N+12]}${ABC[N+24:N+28]}${ABC[N+12:N+16]}${ABC[N+28:N+32]}, 1)), _mm256_set_epi32(7, 3, 5, 1, 6, 2, 4, 0));
75      $else:
76        __m128i vout${ABC[N:N+16]} = _mm_shuffle_epi32(${_MM_PACKXS_EPI16}(_mm256_castsi256_si128(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}), _mm256_extracti128_si256(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));
77
78    $for N in range(0, BATCH_TILE, 32):
79      $if N + 16 < BATCH_TILE:
80        vout${ABC[N:N+32]} = ${_MM256_MAX_EPX8}(vout${ABC[N:N+32]}, voutput_min);
81      $elif BATCH_TILE > 16:
82        vout${ABC[N:N+16]} = ${_MM_MAX_EPX8}(vout${ABC[N:N+16]}, _mm256_castsi256_si128(voutput_min));
83      $else:
84        vout${ABC[N:N+16]} = ${_MM_MAX_EPX8}(vout${ABC[N:N+16]}, voutput_min);
85
86    $for N in range(0, BATCH_TILE, 32):
87      $if N + 16 < BATCH_TILE:
88        vout${ABC[N:N+32]} = ${_MM256_MIN_EPX8}(vout${ABC[N:N+32]}, voutput_max);
89      $elif BATCH_TILE > 16:
90        vout${ABC[N:N+16]} = ${_MM_MIN_EPX8}(vout${ABC[N:N+16]}, _mm256_castsi256_si128(voutput_max));
91      $else:
92        vout${ABC[N:N+16]} = ${_MM_MIN_EPX8}(vout${ABC[N:N+16]}, voutput_max);
93
94    $if BATCH_TILE >= 32:
95      _mm256_storeu_si256((__m256i*) output, vout${ABC[0:32]});
96    $else:
97      _mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
98    $for N in range(32, BATCH_TILE, 32):
99      $if N + 8 < BATCH_TILE:
100        _mm256_storeu_si256((__m256i*) (output + ${N}), vout${ABC[N:N+32]});
101      $else:
102        _mm_storeu_si128((__m128i*) (output + ${N}), vout${ABC[N:N+16]});
103    output += ${BATCH_TILE};
104  }
105  if XNN_UNLIKELY(n != 0) {
106    ${"do " if BATCH_TILE > 16 else ""}{
107      $if BATCH_TILE > 16:
108        const __m512i va${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) input_a));
109        const __m512i vb${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) input_b));
110        input_a += 16;
111        input_b += 16;
112      $else:
113        const __mmask16 vmask = _cvtu32_mask16((uint32_t) ((UINT32_C(1) << n) - UINT32_C(1)));
114        const __m512i va${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_maskz_loadu_epi8(vmask, input_a));
115        const __m512i vb${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_maskz_loadu_epi8(vmask, input_b));
116
117      __m512i vacc${ABC[0:16]} = _mm512_add_epi32(vbias, _mm512_mullo_epi32(va${ABC[0:16]}, va_multiplier));
118
119      vacc${ABC[0:16]} = _mm512_add_epi32(vacc${ABC[0:16]}, _mm512_mullo_epi32(vb${ABC[0:16]}, vb_multiplier));
120
121      vacc${ABC[0:16]} = _mm512_sra_epi32(vacc${ABC[0:16]}, vshift);
122
123      $if BATCH_TILE > 16:
124        __m256i vout${ABC[0:4]}${ABC[8:12]}${ABC[4:8]}${ABC[12:16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[0:16]}), _mm512_extracti32x8_epi32(vacc${ABC[0:16]}, 1)), _mm512_castsi512_si256(voutput_zero_point));
125      $else:
126        __m256i vout${ABC[0:4]}${ABC[8:12]}${ABC[4:8]}${ABC[12:16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[0:16]}), _mm512_extracti32x8_epi32(vacc${ABC[0:16]}, 1)), voutput_zero_point);
127      __m128i vout${ABC[0:16]} = _mm_shuffle_epi32(${_MM_PACKXS_EPI16}(_mm256_castsi256_si128(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}), _mm256_extracti128_si256(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));
128      $if BATCH_TILE > 16:
129        vout${ABC[0:16]} = ${_MM_MAX_EPX8}(vout${ABC[0:16]}, _mm256_castsi256_si128(voutput_min));
130        vout${ABC[0:16]} = ${_MM_MIN_EPX8}(vout${ABC[0:16]}, _mm256_castsi256_si128(voutput_max));
131      $else:
132        vout${ABC[0:16]} = ${_MM_MAX_EPX8}(vout${ABC[0:16]}, voutput_min);
133        vout${ABC[0:16]} = ${_MM_MIN_EPX8}(vout${ABC[0:16]}, voutput_max);
134
135      $if BATCH_TILE > 16:
136        if XNN_LIKELY(n >= (16 * sizeof(${XINT8_T}))) {
137          _mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
138          output += 16;
139          n -= 16 * sizeof(${XINT8_T});
140        } else {
141          const __mmask16 vmask = _cvtu32_mask16((uint32_t) ((UINT32_C(1) << n) - UINT32_C(1)));
142          _mm_mask_storeu_epi8(output, vmask, vout${ABC[0:16]});
143          n = 0;
144        }
145      $else:
146        _mm_mask_storeu_epi8(output, vmask, vout${ABC[0:16]});
147    }${" while (n != 0);" if BATCH_TILE > 16 else ""}
148  }
149}
150