1 // Copyright 2020 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <emmintrin.h>
9
10 #include <xnnpack/fill.h>
11 #include <xnnpack/unaligned.h>
12
13
xnn_xx_fill_ukernel__sse2_x64(size_t rows,size_t channels,void * output,size_t output_stride,const uint32_t fill_pattern)14 void xnn_xx_fill_ukernel__sse2_x64(
15 size_t rows,
16 size_t channels,
17 void* output,
18 size_t output_stride,
19 const uint32_t fill_pattern)
20 {
21 assert(rows != 0);
22 assert(channels != 0);
23
24 const size_t output_increment = output_stride - channels;
25
26 const __m128i vfill = _mm_shuffle_epi32(_mm_cvtsi32_si128(fill_pattern), _MM_SHUFFLE(0, 0, 0, 0));
27 do {
28 size_t c = channels;
29 for (; c >= 64 * sizeof(uint8_t); c -= 64 * sizeof(uint8_t)) {
30 _mm_storeu_si128((__m128i*) output, vfill);
31 _mm_storeu_si128((__m128i*) output + 1, vfill);
32 _mm_storeu_si128((__m128i*) output + 2, vfill);
33 _mm_storeu_si128((__m128i*) output + 3, vfill);
34 output = ((uint8_t*) output + 64);
35 }
36 for (; c >= 16 * sizeof(uint8_t); c -= 16 * sizeof(uint8_t)) {
37 _mm_storeu_si128((__m128i*) output, vfill);
38 output = ((uint8_t*) output + 16);
39 }
40 if XNN_UNLIKELY(c != 0) {
41 if XNN_LIKELY(c & (8 * sizeof(uint8_t))) {
42 _mm_storel_epi64(output, vfill);
43 output = ((uint8_t*) output + 8);
44 }
45 if XNN_LIKELY(c & (4 * sizeof(uint8_t))) {
46 unaligned_store_u32(output, fill_pattern);
47 output = ((uint8_t*) output + 4);
48 }
49 uint32_t vfill_subpattern = fill_pattern;
50 if XNN_LIKELY(c & (2 * sizeof(uint8_t))) {
51 unaligned_store_u16(output, (uint16_t) vfill_subpattern);
52 vfill_subpattern >>= 16;
53 output = ((uint8_t*) output + 2);
54 }
55 if XNN_LIKELY(c & (1 * sizeof(uint8_t))) {
56 *((uint8_t*) output) = (uint8_t) vfill_subpattern;
57 output = ((uint8_t*) output + 1);
58 }
59 }
60 output = (void*) ((uintptr_t) output + output_increment);
61 } while (--rows != 0);
62 }
63