1 /*
2 * Copyright (c) 2023 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12 #include <stdint.h>
13
14 #include "./vpx_dsp_rtcd.h"
15 #include "vpx_dsp/arm/sum_neon.h"
16
highbd_sse_8x1_init_neon(const uint16_t * src,const uint16_t * ref,uint32x4_t * sse_acc0,uint32x4_t * sse_acc1)17 static INLINE void highbd_sse_8x1_init_neon(const uint16_t *src,
18 const uint16_t *ref,
19 uint32x4_t *sse_acc0,
20 uint32x4_t *sse_acc1) {
21 uint16x8_t s = vld1q_u16(src);
22 uint16x8_t r = vld1q_u16(ref);
23
24 uint16x8_t abs_diff = vabdq_u16(s, r);
25 uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
26 uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
27
28 *sse_acc0 = vmull_u16(abs_diff_lo, abs_diff_lo);
29 *sse_acc1 = vmull_u16(abs_diff_hi, abs_diff_hi);
30 }
31
highbd_sse_8x1_neon(const uint16_t * src,const uint16_t * ref,uint32x4_t * sse_acc0,uint32x4_t * sse_acc1)32 static INLINE void highbd_sse_8x1_neon(const uint16_t *src, const uint16_t *ref,
33 uint32x4_t *sse_acc0,
34 uint32x4_t *sse_acc1) {
35 uint16x8_t s = vld1q_u16(src);
36 uint16x8_t r = vld1q_u16(ref);
37
38 uint16x8_t abs_diff = vabdq_u16(s, r);
39 uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
40 uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
41
42 *sse_acc0 = vmlal_u16(*sse_acc0, abs_diff_lo, abs_diff_lo);
43 *sse_acc1 = vmlal_u16(*sse_acc1, abs_diff_hi, abs_diff_hi);
44 }
45
highbd_sse_64xh_neon(const uint16_t * src,int src_stride,const uint16_t * ref,int ref_stride,int height)46 static INLINE int64_t highbd_sse_64xh_neon(const uint16_t *src, int src_stride,
47 const uint16_t *ref, int ref_stride,
48 int height) {
49 uint32x4_t sse[8];
50 highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
51 highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
52 highbd_sse_8x1_init_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
53 highbd_sse_8x1_init_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
54 highbd_sse_8x1_neon(src + 4 * 8, ref + 4 * 8, &sse[0], &sse[1]);
55 highbd_sse_8x1_neon(src + 5 * 8, ref + 5 * 8, &sse[2], &sse[3]);
56 highbd_sse_8x1_neon(src + 6 * 8, ref + 6 * 8, &sse[4], &sse[5]);
57 highbd_sse_8x1_neon(src + 7 * 8, ref + 7 * 8, &sse[6], &sse[7]);
58
59 src += src_stride;
60 ref += ref_stride;
61
62 while (--height != 0) {
63 highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
64 highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
65 highbd_sse_8x1_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
66 highbd_sse_8x1_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
67 highbd_sse_8x1_neon(src + 4 * 8, ref + 4 * 8, &sse[0], &sse[1]);
68 highbd_sse_8x1_neon(src + 5 * 8, ref + 5 * 8, &sse[2], &sse[3]);
69 highbd_sse_8x1_neon(src + 6 * 8, ref + 6 * 8, &sse[4], &sse[5]);
70 highbd_sse_8x1_neon(src + 7 * 8, ref + 7 * 8, &sse[6], &sse[7]);
71
72 src += src_stride;
73 ref += ref_stride;
74 }
75
76 return horizontal_long_add_uint32x4_x8(sse);
77 }
78
highbd_sse_32xh_neon(const uint16_t * src,int src_stride,const uint16_t * ref,int ref_stride,int height)79 static INLINE int64_t highbd_sse_32xh_neon(const uint16_t *src, int src_stride,
80 const uint16_t *ref, int ref_stride,
81 int height) {
82 uint32x4_t sse[8];
83 highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
84 highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
85 highbd_sse_8x1_init_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
86 highbd_sse_8x1_init_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
87
88 src += src_stride;
89 ref += ref_stride;
90
91 while (--height != 0) {
92 highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
93 highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
94 highbd_sse_8x1_neon(src + 2 * 8, ref + 2 * 8, &sse[4], &sse[5]);
95 highbd_sse_8x1_neon(src + 3 * 8, ref + 3 * 8, &sse[6], &sse[7]);
96
97 src += src_stride;
98 ref += ref_stride;
99 }
100
101 return horizontal_long_add_uint32x4_x8(sse);
102 }
103
highbd_sse_16xh_neon(const uint16_t * src,int src_stride,const uint16_t * ref,int ref_stride,int height)104 static INLINE int64_t highbd_sse_16xh_neon(const uint16_t *src, int src_stride,
105 const uint16_t *ref, int ref_stride,
106 int height) {
107 uint32x4_t sse[4];
108 highbd_sse_8x1_init_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
109 highbd_sse_8x1_init_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
110
111 src += src_stride;
112 ref += ref_stride;
113
114 while (--height != 0) {
115 highbd_sse_8x1_neon(src + 0 * 8, ref + 0 * 8, &sse[0], &sse[1]);
116 highbd_sse_8x1_neon(src + 1 * 8, ref + 1 * 8, &sse[2], &sse[3]);
117
118 src += src_stride;
119 ref += ref_stride;
120 }
121
122 return horizontal_long_add_uint32x4_x4(sse);
123 }
124
highbd_sse_8xh_neon(const uint16_t * src,int src_stride,const uint16_t * ref,int ref_stride,int height)125 static INLINE int64_t highbd_sse_8xh_neon(const uint16_t *src, int src_stride,
126 const uint16_t *ref, int ref_stride,
127 int height) {
128 uint32x4_t sse[2];
129 highbd_sse_8x1_init_neon(src, ref, &sse[0], &sse[1]);
130
131 src += src_stride;
132 ref += ref_stride;
133
134 while (--height != 0) {
135 highbd_sse_8x1_neon(src, ref, &sse[0], &sse[1]);
136
137 src += src_stride;
138 ref += ref_stride;
139 }
140
141 return horizontal_long_add_uint32x4_x2(sse);
142 }
143
highbd_sse_4xh_neon(const uint16_t * src,int src_stride,const uint16_t * ref,int ref_stride,int height)144 static INLINE int64_t highbd_sse_4xh_neon(const uint16_t *src, int src_stride,
145 const uint16_t *ref, int ref_stride,
146 int height) {
147 // Peel the first loop iteration.
148 uint16x4_t s = vld1_u16(src);
149 uint16x4_t r = vld1_u16(ref);
150
151 uint16x4_t abs_diff = vabd_u16(s, r);
152 uint32x4_t sse = vmull_u16(abs_diff, abs_diff);
153
154 src += src_stride;
155 ref += ref_stride;
156
157 while (--height != 0) {
158 s = vld1_u16(src);
159 r = vld1_u16(ref);
160
161 abs_diff = vabd_u16(s, r);
162 sse = vmlal_u16(sse, abs_diff, abs_diff);
163
164 src += src_stride;
165 ref += ref_stride;
166 }
167
168 return horizontal_long_add_uint32x4(sse);
169 }
170
highbd_sse_wxh_neon(const uint16_t * src,int src_stride,const uint16_t * ref,int ref_stride,int width,int height)171 static INLINE int64_t highbd_sse_wxh_neon(const uint16_t *src, int src_stride,
172 const uint16_t *ref, int ref_stride,
173 int width, int height) {
174 // { 0, 1, 2, 3, 4, 5, 6, 7 }
175 uint16x8_t k01234567 = vmovl_u8(vcreate_u8(0x0706050403020100));
176 uint16x8_t remainder_mask = vcltq_u16(k01234567, vdupq_n_u16(width & 7));
177 uint64_t sse = 0;
178
179 do {
180 int w = width;
181 int offset = 0;
182
183 do {
184 uint16x8_t s = vld1q_u16(src + offset);
185 uint16x8_t r = vld1q_u16(ref + offset);
186 uint16x8_t abs_diff;
187 uint16x4_t abs_diff_lo;
188 uint16x4_t abs_diff_hi;
189 uint32x4_t sse_u32;
190
191 if (w < 8) {
192 // Mask out-of-range elements.
193 s = vandq_u16(s, remainder_mask);
194 r = vandq_u16(r, remainder_mask);
195 }
196
197 abs_diff = vabdq_u16(s, r);
198 abs_diff_lo = vget_low_u16(abs_diff);
199 abs_diff_hi = vget_high_u16(abs_diff);
200
201 sse_u32 = vmull_u16(abs_diff_lo, abs_diff_lo);
202 sse_u32 = vmlal_u16(sse_u32, abs_diff_hi, abs_diff_hi);
203
204 sse += horizontal_long_add_uint32x4(sse_u32);
205
206 offset += 8;
207 w -= 8;
208 } while (w > 0);
209
210 src += src_stride;
211 ref += ref_stride;
212 } while (--height != 0);
213
214 return sse;
215 }
216
vpx_highbd_sse_neon(const uint8_t * src8,int src_stride,const uint8_t * ref8,int ref_stride,int width,int height)217 int64_t vpx_highbd_sse_neon(const uint8_t *src8, int src_stride,
218 const uint8_t *ref8, int ref_stride, int width,
219 int height) {
220 uint16_t *src = CONVERT_TO_SHORTPTR(src8);
221 uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
222
223 switch (width) {
224 case 4:
225 return highbd_sse_4xh_neon(src, src_stride, ref, ref_stride, height);
226 case 8:
227 return highbd_sse_8xh_neon(src, src_stride, ref, ref_stride, height);
228 case 16:
229 return highbd_sse_16xh_neon(src, src_stride, ref, ref_stride, height);
230 case 32:
231 return highbd_sse_32xh_neon(src, src_stride, ref, ref_stride, height);
232 case 64:
233 return highbd_sse_64xh_neon(src, src_stride, ref, ref_stride, height);
234 default:
235 return highbd_sse_wxh_neon(src, src_stride, ref, ref_stride, width,
236 height);
237 }
238 }
239