1 /*
2 * Copyright(c) 2019 Intel Corporation
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at https://www.aomedia.org/license/software-license. If the
8 * Alliance for Open Media Patent License 1.0 was not distributed with this
9 * source code in the PATENTS file, you can obtain it at
10 * https://www.aomedia.org/license/patent-license.
11 */
12
13 #ifndef AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_AVX2_H_
14 #define AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_AVX2_H_
15
16 #include <immintrin.h>
17
18 #include "config/aom_config.h"
19
20 #include "aom/aom_integer.h"
21
22 #ifndef _mm256_set_m128i
23 #define _mm256_set_m128i(/* __m128i */ hi, /* __m128i */ lo) \
24 _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 0x1)
25 #endif
26
27 #ifndef _mm256_setr_m128i
28 #define _mm256_setr_m128i(/* __m128i */ lo, /* __m128i */ hi) \
29 _mm256_set_m128i((hi), (lo))
30 #endif
31
load_u8_4x2_avx2(const uint8_t * const src,const ptrdiff_t stride)32 static inline __m256i load_u8_4x2_avx2(const uint8_t *const src,
33 const ptrdiff_t stride) {
34 __m128i src01;
35 src01 = _mm_cvtsi32_si128(*(int32_t *)(src + 0 * stride));
36 src01 = _mm_insert_epi32(src01, *(int32_t *)(src + 1 * stride), 1);
37 return _mm256_setr_m128i(src01, _mm_setzero_si128());
38 }
39
load_u8_4x4_avx2(const uint8_t * const src,const ptrdiff_t stride)40 static inline __m256i load_u8_4x4_avx2(const uint8_t *const src,
41 const ptrdiff_t stride) {
42 __m128i src01, src23;
43 src01 = _mm_cvtsi32_si128(*(int32_t *)(src + 0 * stride));
44 src01 = _mm_insert_epi32(src01, *(int32_t *)(src + 1 * stride), 1);
45 src23 = _mm_cvtsi32_si128(*(int32_t *)(src + 2 * stride));
46 src23 = _mm_insert_epi32(src23, *(int32_t *)(src + 3 * stride), 1);
47 return _mm256_setr_m128i(src01, src23);
48 }
49
load_u8_8x2_avx2(const uint8_t * const src,const ptrdiff_t stride)50 static inline __m256i load_u8_8x2_avx2(const uint8_t *const src,
51 const ptrdiff_t stride) {
52 const __m128i src0 = _mm_loadl_epi64((__m128i *)(src + 0 * stride));
53 const __m128i src1 = _mm_loadl_epi64((__m128i *)(src + 1 * stride));
54 return _mm256_setr_m128i(src0, src1);
55 }
56
load_u8_8x4_avx2(const uint8_t * const src,const ptrdiff_t stride)57 static inline __m256i load_u8_8x4_avx2(const uint8_t *const src,
58 const ptrdiff_t stride) {
59 __m128i src01, src23;
60 src01 = _mm_loadl_epi64((__m128i *)(src + 0 * stride));
61 src01 = _mm_castpd_si128(_mm_loadh_pd(_mm_castsi128_pd(src01),
62 (double *)(void *)(src + 1 * stride)));
63 src23 = _mm_loadl_epi64((__m128i *)(src + 2 * stride));
64 src23 = _mm_castpd_si128(_mm_loadh_pd(_mm_castsi128_pd(src23),
65 (double *)(void *)(src + 3 * stride)));
66 return _mm256_setr_m128i(src01, src23);
67 }
68
loadu_8bit_16x2_avx2(const void * const src,const ptrdiff_t strideInByte)69 static inline __m256i loadu_8bit_16x2_avx2(const void *const src,
70 const ptrdiff_t strideInByte) {
71 const __m128i src0 = _mm_loadu_si128((__m128i *)src);
72 const __m128i src1 =
73 _mm_loadu_si128((__m128i *)((uint8_t *)src + strideInByte));
74 return _mm256_setr_m128i(src0, src1);
75 }
76
loadu_u8_16x2_avx2(const uint8_t * const src,const ptrdiff_t stride)77 static inline __m256i loadu_u8_16x2_avx2(const uint8_t *const src,
78 const ptrdiff_t stride) {
79 return loadu_8bit_16x2_avx2(src, sizeof(*src) * stride);
80 }
81
loadu_u16_8x2_avx2(const uint16_t * const src,const ptrdiff_t stride)82 static inline __m256i loadu_u16_8x2_avx2(const uint16_t *const src,
83 const ptrdiff_t stride) {
84 return loadu_8bit_16x2_avx2(src, sizeof(*src) * stride);
85 }
86
storeu_8bit_16x2_avx2(const __m256i src,void * const dst,const ptrdiff_t strideInByte)87 static inline void storeu_8bit_16x2_avx2(const __m256i src, void *const dst,
88 const ptrdiff_t strideInByte) {
89 const __m128i d0 = _mm256_castsi256_si128(src);
90 const __m128i d1 = _mm256_extracti128_si256(src, 1);
91 _mm_storeu_si128((__m128i *)dst, d0);
92 _mm_storeu_si128((__m128i *)((uint8_t *)dst + strideInByte), d1);
93 }
94
storeu_u8_16x2_avx2(const __m256i src,uint8_t * const dst,const ptrdiff_t stride)95 static inline void storeu_u8_16x2_avx2(const __m256i src, uint8_t *const dst,
96 const ptrdiff_t stride) {
97 storeu_8bit_16x2_avx2(src, dst, sizeof(*dst) * stride);
98 }
99
storeu_s16_8x2_avx2(const __m256i src,int16_t * const dst,const ptrdiff_t stride)100 static inline void storeu_s16_8x2_avx2(const __m256i src, int16_t *const dst,
101 const ptrdiff_t stride) {
102 storeu_8bit_16x2_avx2(src, dst, sizeof(*dst) * stride);
103 }
104
storeu_u16_8x2_avx2(const __m256i src,uint16_t * const dst,const ptrdiff_t stride)105 static inline void storeu_u16_8x2_avx2(const __m256i src, uint16_t *const dst,
106 const ptrdiff_t stride) {
107 storeu_8bit_16x2_avx2(src, dst, sizeof(*dst) * stride);
108 }
109
110 #endif // AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_AVX2_H_
111