1 /*
2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11 #include <immintrin.h>
12
13 #include "config/aom_dsp_rtcd.h"
14
subtract32_avx2(int16_t * diff_ptr,const uint8_t * src_ptr,const uint8_t * pred_ptr)15 static inline void subtract32_avx2(int16_t *diff_ptr, const uint8_t *src_ptr,
16 const uint8_t *pred_ptr) {
17 __m256i s = _mm256_lddqu_si256((__m256i *)(src_ptr));
18 __m256i p = _mm256_lddqu_si256((__m256i *)(pred_ptr));
19 __m256i set_one_minusone = _mm256_set1_epi32((int)0xff01ff01);
20 __m256i diff0 = _mm256_unpacklo_epi8(s, p);
21 __m256i diff1 = _mm256_unpackhi_epi8(s, p);
22 diff0 = _mm256_maddubs_epi16(diff0, set_one_minusone);
23 diff1 = _mm256_maddubs_epi16(diff1, set_one_minusone);
24 _mm256_store_si256((__m256i *)(diff_ptr),
25 _mm256_permute2x128_si256(diff0, diff1, 0x20));
26 _mm256_store_si256((__m256i *)(diff_ptr + 16),
27 _mm256_permute2x128_si256(diff0, diff1, 0x31));
28 }
29
subtract_block_16xn_avx2(int rows,int16_t * diff_ptr,ptrdiff_t diff_stride,const uint8_t * src_ptr,ptrdiff_t src_stride,const uint8_t * pred_ptr,ptrdiff_t pred_stride)30 static inline void subtract_block_16xn_avx2(
31 int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
32 ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
33 for (int32_t j = 0; j < rows; ++j) {
34 __m128i s = _mm_lddqu_si128((__m128i *)(src_ptr));
35 __m128i p = _mm_lddqu_si128((__m128i *)(pred_ptr));
36 __m256i s_0 = _mm256_cvtepu8_epi16(s);
37 __m256i p_0 = _mm256_cvtepu8_epi16(p);
38 const __m256i d_0 = _mm256_sub_epi16(s_0, p_0);
39 _mm256_store_si256((__m256i *)(diff_ptr), d_0);
40 src_ptr += src_stride;
41 pred_ptr += pred_stride;
42 diff_ptr += diff_stride;
43 }
44 }
45
subtract_block_32xn_avx2(int rows,int16_t * diff_ptr,ptrdiff_t diff_stride,const uint8_t * src_ptr,ptrdiff_t src_stride,const uint8_t * pred_ptr,ptrdiff_t pred_stride)46 static inline void subtract_block_32xn_avx2(
47 int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
48 ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
49 for (int32_t j = 0; j < rows; ++j) {
50 subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
51 src_ptr += src_stride;
52 pred_ptr += pred_stride;
53 diff_ptr += diff_stride;
54 }
55 }
56
subtract_block_64xn_avx2(int rows,int16_t * diff_ptr,ptrdiff_t diff_stride,const uint8_t * src_ptr,ptrdiff_t src_stride,const uint8_t * pred_ptr,ptrdiff_t pred_stride)57 static inline void subtract_block_64xn_avx2(
58 int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
59 ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
60 for (int32_t j = 0; j < rows; ++j) {
61 subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
62 subtract32_avx2(diff_ptr + 32, src_ptr + 32, pred_ptr + 32);
63 src_ptr += src_stride;
64 pred_ptr += pred_stride;
65 diff_ptr += diff_stride;
66 }
67 }
68
subtract_block_128xn_avx2(int rows,int16_t * diff_ptr,ptrdiff_t diff_stride,const uint8_t * src_ptr,ptrdiff_t src_stride,const uint8_t * pred_ptr,ptrdiff_t pred_stride)69 static inline void subtract_block_128xn_avx2(
70 int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
71 ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
72 for (int32_t j = 0; j < rows; ++j) {
73 subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
74 subtract32_avx2(diff_ptr + 32, src_ptr + 32, pred_ptr + 32);
75 subtract32_avx2(diff_ptr + 64, src_ptr + 64, pred_ptr + 64);
76 subtract32_avx2(diff_ptr + 96, src_ptr + 96, pred_ptr + 96);
77 src_ptr += src_stride;
78 pred_ptr += pred_stride;
79 diff_ptr += diff_stride;
80 }
81 }
82
aom_subtract_block_avx2(int rows,int cols,int16_t * diff_ptr,ptrdiff_t diff_stride,const uint8_t * src_ptr,ptrdiff_t src_stride,const uint8_t * pred_ptr,ptrdiff_t pred_stride)83 void aom_subtract_block_avx2(int rows, int cols, int16_t *diff_ptr,
84 ptrdiff_t diff_stride, const uint8_t *src_ptr,
85 ptrdiff_t src_stride, const uint8_t *pred_ptr,
86 ptrdiff_t pred_stride) {
87 switch (cols) {
88 case 16:
89 subtract_block_16xn_avx2(rows, diff_ptr, diff_stride, src_ptr, src_stride,
90 pred_ptr, pred_stride);
91 break;
92 case 32:
93 subtract_block_32xn_avx2(rows, diff_ptr, diff_stride, src_ptr, src_stride,
94 pred_ptr, pred_stride);
95 break;
96 case 64:
97 subtract_block_64xn_avx2(rows, diff_ptr, diff_stride, src_ptr, src_stride,
98 pred_ptr, pred_stride);
99 break;
100 case 128:
101 subtract_block_128xn_avx2(rows, diff_ptr, diff_stride, src_ptr,
102 src_stride, pred_ptr, pred_stride);
103 break;
104 default:
105 aom_subtract_block_sse2(rows, cols, diff_ptr, diff_stride, src_ptr,
106 src_stride, pred_ptr, pred_stride);
107 break;
108 }
109 }
110