1 /*
2 * Copyright (c) 2024 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #ifndef VPX_VPX_DSP_ARM_HIGHBD_CONVOLVE8_SVE_H_
12 #define VPX_VPX_DSP_ARM_HIGHBD_CONVOLVE8_SVE_H_
13
14 #include <arm_neon.h>
15
16 #include "vpx_dsp/arm/vpx_neon_sve_bridge.h"
17
highbd_convolve4_4_sve(const int16x4_t s[4],const int16x8_t filter,const uint16x4_t max)18 static INLINE uint16x4_t highbd_convolve4_4_sve(const int16x4_t s[4],
19 const int16x8_t filter,
20 const uint16x4_t max) {
21 int16x8_t s01 = vcombine_s16(s[0], s[1]);
22 int16x8_t s23 = vcombine_s16(s[2], s[3]);
23
24 int64x2_t sum01 = vpx_dotq_lane_s16(vdupq_n_s64(0), s01, filter, 0);
25 int64x2_t sum23 = vpx_dotq_lane_s16(vdupq_n_s64(0), s23, filter, 0);
26
27 int32x4_t res_s32 = vcombine_s32(vmovn_s64(sum01), vmovn_s64(sum23));
28
29 uint16x4_t res_u16 = vqrshrun_n_s32(res_s32, FILTER_BITS);
30 return vmin_u16(res_u16, max);
31 }
32
highbd_convolve4_8_sve(const int16x8_t s[4],const int16x8_t filter,const uint16x8_t max,uint16x8_t idx)33 static INLINE uint16x8_t highbd_convolve4_8_sve(const int16x8_t s[4],
34 const int16x8_t filter,
35 const uint16x8_t max,
36 uint16x8_t idx) {
37 int64x2_t sum04 = vpx_dotq_lane_s16(vdupq_n_s64(0), s[0], filter, 0);
38 int64x2_t sum15 = vpx_dotq_lane_s16(vdupq_n_s64(0), s[1], filter, 0);
39 int64x2_t sum26 = vpx_dotq_lane_s16(vdupq_n_s64(0), s[2], filter, 0);
40 int64x2_t sum37 = vpx_dotq_lane_s16(vdupq_n_s64(0), s[3], filter, 0);
41
42 int32x4_t res0 = vcombine_s32(vmovn_s64(sum04), vmovn_s64(sum15));
43 int32x4_t res1 = vcombine_s32(vmovn_s64(sum26), vmovn_s64(sum37));
44
45 uint16x8_t res = vcombine_u16(vqrshrun_n_s32(res0, FILTER_BITS),
46 vqrshrun_n_s32(res1, FILTER_BITS));
47
48 res = vpx_tbl_u16(res, idx);
49
50 return vminq_u16(res, max);
51 }
52
highbd_convolve8_4(const int16x8_t s[4],const int16x8_t filter,const uint16x4_t max)53 static INLINE uint16x4_t highbd_convolve8_4(const int16x8_t s[4],
54 const int16x8_t filter,
55 const uint16x4_t max) {
56 int64x2_t sum[4];
57
58 sum[0] = vpx_dotq_s16(vdupq_n_s64(0), s[0], filter);
59 sum[1] = vpx_dotq_s16(vdupq_n_s64(0), s[1], filter);
60 sum[2] = vpx_dotq_s16(vdupq_n_s64(0), s[2], filter);
61 sum[3] = vpx_dotq_s16(vdupq_n_s64(0), s[3], filter);
62
63 sum[0] = vpaddq_s64(sum[0], sum[1]);
64 sum[2] = vpaddq_s64(sum[2], sum[3]);
65
66 int32x4_t res_s32 = vcombine_s32(vmovn_s64(sum[0]), vmovn_s64(sum[2]));
67
68 uint16x4_t res_u16 = vqrshrun_n_s32(res_s32, FILTER_BITS);
69 return vmin_u16(res_u16, max);
70 }
71
highbd_convolve8_8(const int16x8_t s[8],const int16x8_t filter,const uint16x8_t max)72 static INLINE uint16x8_t highbd_convolve8_8(const int16x8_t s[8],
73 const int16x8_t filter,
74 const uint16x8_t max) {
75 int64x2_t sum[8];
76
77 sum[0] = vpx_dotq_s16(vdupq_n_s64(0), s[0], filter);
78 sum[1] = vpx_dotq_s16(vdupq_n_s64(0), s[1], filter);
79 sum[2] = vpx_dotq_s16(vdupq_n_s64(0), s[2], filter);
80 sum[3] = vpx_dotq_s16(vdupq_n_s64(0), s[3], filter);
81 sum[4] = vpx_dotq_s16(vdupq_n_s64(0), s[4], filter);
82 sum[5] = vpx_dotq_s16(vdupq_n_s64(0), s[5], filter);
83 sum[6] = vpx_dotq_s16(vdupq_n_s64(0), s[6], filter);
84 sum[7] = vpx_dotq_s16(vdupq_n_s64(0), s[7], filter);
85
86 int64x2_t sum01 = vpaddq_s64(sum[0], sum[1]);
87 int64x2_t sum23 = vpaddq_s64(sum[2], sum[3]);
88 int64x2_t sum45 = vpaddq_s64(sum[4], sum[5]);
89 int64x2_t sum67 = vpaddq_s64(sum[6], sum[7]);
90
91 int32x4_t res0 = vcombine_s32(vmovn_s64(sum01), vmovn_s64(sum23));
92 int32x4_t res1 = vcombine_s32(vmovn_s64(sum45), vmovn_s64(sum67));
93
94 uint16x8_t res = vcombine_u16(vqrshrun_n_s32(res0, FILTER_BITS),
95 vqrshrun_n_s32(res1, FILTER_BITS));
96 return vminq_u16(res, max);
97 }
98
99 #endif // VPX_VPX_DSP_ARM_HIGHBD_CONVOLVE8_SVE_H_
100