1 /*
2 * Copyright (c) 2014 The WebM project authors. All rights reserved.
3 * Copyright (c) 2023, Alliance for Open Media. All rights reserved.
4 *
5 * This source code is subject to the terms of the BSD 2 Clause License and
6 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
7 * was not distributed with this source code in the LICENSE file, you can
8 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
9 * Media Patent License 1.0 was not distributed with this source code in the
10 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
11 */
12
13 #include <arm_neon.h>
14 #include <assert.h>
15
16 #include "config/aom_config.h"
17 #include "config/aom_dsp_rtcd.h"
18
19 #include "aom/aom_integer.h"
20 #include "aom_dsp/aom_dsp_common.h"
21 #include "aom_dsp/aom_filter.h"
22 #include "aom_dsp/arm/aom_filter.h"
23 #include "aom_dsp/arm/highbd_convolve8_neon.h"
24 #include "aom_dsp/arm/mem_neon.h"
25 #include "aom_dsp/arm/transpose_neon.h"
26 #include "aom_ports/mem.h"
27
highbd_convolve8_4(const int16x4_t s0,const int16x4_t s1,const int16x4_t s2,const int16x4_t s3,const int16x4_t s4,const int16x4_t s5,const int16x4_t s6,const int16x4_t s7,const int16x8_t filter,const uint16x4_t max)28 static inline uint16x4_t highbd_convolve8_4(
29 const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
30 const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
31 const int16x4_t s6, const int16x4_t s7, const int16x8_t filter,
32 const uint16x4_t max) {
33 const int16x4_t filter_lo = vget_low_s16(filter);
34 const int16x4_t filter_hi = vget_high_s16(filter);
35
36 int32x4_t sum = vmull_lane_s16(s0, filter_lo, 0);
37 sum = vmlal_lane_s16(sum, s1, filter_lo, 1);
38 sum = vmlal_lane_s16(sum, s2, filter_lo, 2);
39 sum = vmlal_lane_s16(sum, s3, filter_lo, 3);
40 sum = vmlal_lane_s16(sum, s4, filter_hi, 0);
41 sum = vmlal_lane_s16(sum, s5, filter_hi, 1);
42 sum = vmlal_lane_s16(sum, s6, filter_hi, 2);
43 sum = vmlal_lane_s16(sum, s7, filter_hi, 3);
44
45 uint16x4_t res = vqrshrun_n_s32(sum, FILTER_BITS);
46
47 return vmin_u16(res, max);
48 }
49
highbd_convolve8_8(const int16x8_t s0,const int16x8_t s1,const int16x8_t s2,const int16x8_t s3,const int16x8_t s4,const int16x8_t s5,const int16x8_t s6,const int16x8_t s7,const int16x8_t filter,const uint16x8_t max)50 static inline uint16x8_t highbd_convolve8_8(
51 const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
52 const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
53 const int16x8_t s6, const int16x8_t s7, const int16x8_t filter,
54 const uint16x8_t max) {
55 const int16x4_t filter_lo = vget_low_s16(filter);
56 const int16x4_t filter_hi = vget_high_s16(filter);
57
58 int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), filter_lo, 0);
59 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), filter_lo, 1);
60 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), filter_lo, 2);
61 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), filter_lo, 3);
62 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), filter_hi, 0);
63 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), filter_hi, 1);
64 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), filter_hi, 2);
65 sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), filter_hi, 3);
66
67 int32x4_t sum1 = vmull_lane_s16(vget_high_s16(s0), filter_lo, 0);
68 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), filter_lo, 1);
69 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), filter_lo, 2);
70 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), filter_lo, 3);
71 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), filter_hi, 0);
72 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), filter_hi, 1);
73 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), filter_hi, 2);
74 sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), filter_hi, 3);
75
76 uint16x8_t res = vcombine_u16(vqrshrun_n_s32(sum0, FILTER_BITS),
77 vqrshrun_n_s32(sum1, FILTER_BITS));
78
79 return vminq_u16(res, max);
80 }
81
highbd_convolve_horiz_8tap_neon(const uint16_t * src_ptr,ptrdiff_t src_stride,uint16_t * dst_ptr,ptrdiff_t dst_stride,const int16_t * x_filter_ptr,int w,int h,int bd)82 static void highbd_convolve_horiz_8tap_neon(
83 const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr,
84 ptrdiff_t dst_stride, const int16_t *x_filter_ptr, int w, int h, int bd) {
85 assert(w >= 4 && h >= 4);
86 const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
87
88 if (w == 4) {
89 const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
90 const int16_t *s = (const int16_t *)src_ptr;
91 uint16_t *d = dst_ptr;
92
93 do {
94 int16x4_t s0[8], s1[8], s2[8], s3[8];
95 load_s16_4x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
96 &s0[4], &s0[5], &s0[6], &s0[7]);
97 load_s16_4x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
98 &s1[4], &s1[5], &s1[6], &s1[7]);
99 load_s16_4x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
100 &s2[4], &s2[5], &s2[6], &s2[7]);
101 load_s16_4x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
102 &s3[4], &s3[5], &s3[6], &s3[7]);
103
104 uint16x4_t d0 = highbd_convolve8_4(s0[0], s0[1], s0[2], s0[3], s0[4],
105 s0[5], s0[6], s0[7], x_filter, max);
106 uint16x4_t d1 = highbd_convolve8_4(s1[0], s1[1], s1[2], s1[3], s1[4],
107 s1[5], s1[6], s1[7], x_filter, max);
108 uint16x4_t d2 = highbd_convolve8_4(s2[0], s2[1], s2[2], s2[3], s2[4],
109 s2[5], s2[6], s2[7], x_filter, max);
110 uint16x4_t d3 = highbd_convolve8_4(s3[0], s3[1], s3[2], s3[3], s3[4],
111 s3[5], s3[6], s3[7], x_filter, max);
112
113 store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
114
115 s += 4 * src_stride;
116 d += 4 * dst_stride;
117 h -= 4;
118 } while (h > 0);
119 } else {
120 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
121 int height = h;
122
123 do {
124 int width = w;
125 const int16_t *s = (const int16_t *)src_ptr;
126 uint16_t *d = dst_ptr;
127
128 do {
129 int16x8_t s0[8], s1[8], s2[8], s3[8];
130 load_s16_8x8(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3],
131 &s0[4], &s0[5], &s0[6], &s0[7]);
132 load_s16_8x8(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3],
133 &s1[4], &s1[5], &s1[6], &s1[7]);
134 load_s16_8x8(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3],
135 &s2[4], &s2[5], &s2[6], &s2[7]);
136 load_s16_8x8(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3],
137 &s3[4], &s3[5], &s3[6], &s3[7]);
138
139 uint16x8_t d0 = highbd_convolve8_8(s0[0], s0[1], s0[2], s0[3], s0[4],
140 s0[5], s0[6], s0[7], x_filter, max);
141 uint16x8_t d1 = highbd_convolve8_8(s1[0], s1[1], s1[2], s1[3], s1[4],
142 s1[5], s1[6], s1[7], x_filter, max);
143 uint16x8_t d2 = highbd_convolve8_8(s2[0], s2[1], s2[2], s2[3], s2[4],
144 s2[5], s2[6], s2[7], x_filter, max);
145 uint16x8_t d3 = highbd_convolve8_8(s3[0], s3[1], s3[2], s3[3], s3[4],
146 s3[5], s3[6], s3[7], x_filter, max);
147
148 store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
149
150 s += 8;
151 d += 8;
152 width -= 8;
153 } while (width > 0);
154 src_ptr += 4 * src_stride;
155 dst_ptr += 4 * dst_stride;
156 height -= 4;
157 } while (height > 0);
158 }
159 }
160
highbd_convolve_horiz_4tap_neon(const uint16_t * src_ptr,ptrdiff_t src_stride,uint16_t * dst_ptr,ptrdiff_t dst_stride,const int16_t * x_filter_ptr,int w,int h,int bd)161 static void highbd_convolve_horiz_4tap_neon(
162 const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr,
163 ptrdiff_t dst_stride, const int16_t *x_filter_ptr, int w, int h, int bd) {
164 assert(w >= 4 && h >= 4);
165 const int16x4_t x_filter = vld1_s16(x_filter_ptr + 2);
166
167 if (w == 4) {
168 const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
169 const int16_t *s = (const int16_t *)src_ptr;
170 uint16_t *d = dst_ptr;
171
172 do {
173 int16x4_t s0[4], s1[4], s2[4], s3[4];
174 load_s16_4x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
175 load_s16_4x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
176 load_s16_4x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
177 load_s16_4x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
178
179 uint16x4_t d0 =
180 highbd_convolve4_4(s0[0], s0[1], s0[2], s0[3], x_filter, max);
181 uint16x4_t d1 =
182 highbd_convolve4_4(s1[0], s1[1], s1[2], s1[3], x_filter, max);
183 uint16x4_t d2 =
184 highbd_convolve4_4(s2[0], s2[1], s2[2], s2[3], x_filter, max);
185 uint16x4_t d3 =
186 highbd_convolve4_4(s3[0], s3[1], s3[2], s3[3], x_filter, max);
187
188 store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
189
190 s += 4 * src_stride;
191 d += 4 * dst_stride;
192 h -= 4;
193 } while (h > 0);
194 } else {
195 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
196 int height = h;
197
198 do {
199 int width = w;
200 const int16_t *s = (const int16_t *)src_ptr;
201 uint16_t *d = dst_ptr;
202
203 do {
204 int16x8_t s0[4], s1[4], s2[4], s3[4];
205 load_s16_8x4(s + 0 * src_stride, 1, &s0[0], &s0[1], &s0[2], &s0[3]);
206 load_s16_8x4(s + 1 * src_stride, 1, &s1[0], &s1[1], &s1[2], &s1[3]);
207 load_s16_8x4(s + 2 * src_stride, 1, &s2[0], &s2[1], &s2[2], &s2[3]);
208 load_s16_8x4(s + 3 * src_stride, 1, &s3[0], &s3[1], &s3[2], &s3[3]);
209
210 uint16x8_t d0 =
211 highbd_convolve4_8(s0[0], s0[1], s0[2], s0[3], x_filter, max);
212 uint16x8_t d1 =
213 highbd_convolve4_8(s1[0], s1[1], s1[2], s1[3], x_filter, max);
214 uint16x8_t d2 =
215 highbd_convolve4_8(s2[0], s2[1], s2[2], s2[3], x_filter, max);
216 uint16x8_t d3 =
217 highbd_convolve4_8(s3[0], s3[1], s3[2], s3[3], x_filter, max);
218
219 store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
220
221 s += 8;
222 d += 8;
223 width -= 8;
224 } while (width > 0);
225 src_ptr += 4 * src_stride;
226 dst_ptr += 4 * dst_stride;
227 height -= 4;
228 } while (height > 0);
229 }
230 }
231
aom_highbd_convolve8_horiz_neon(const uint8_t * src8,ptrdiff_t src_stride,uint8_t * dst8,ptrdiff_t dst_stride,const int16_t * filter_x,int x_step_q4,const int16_t * filter_y,int y_step_q4,int w,int h,int bd)232 void aom_highbd_convolve8_horiz_neon(const uint8_t *src8, ptrdiff_t src_stride,
233 uint8_t *dst8, ptrdiff_t dst_stride,
234 const int16_t *filter_x, int x_step_q4,
235 const int16_t *filter_y, int y_step_q4,
236 int w, int h, int bd) {
237 if (x_step_q4 != 16) {
238 aom_highbd_convolve8_horiz_c(src8, src_stride, dst8, dst_stride, filter_x,
239 x_step_q4, filter_y, y_step_q4, w, h, bd);
240 } else {
241 (void)filter_y;
242 (void)y_step_q4;
243
244 uint16_t *src = CONVERT_TO_SHORTPTR(src8);
245 uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
246
247 src -= SUBPEL_TAPS / 2 - 1;
248
249 const int filter_taps = get_filter_taps_convolve8(filter_x);
250
251 if (filter_taps == 2) {
252 highbd_convolve8_horiz_2tap_neon(src + 3, src_stride, dst, dst_stride,
253 filter_x, w, h, bd);
254 } else if (filter_taps == 4) {
255 highbd_convolve_horiz_4tap_neon(src + 2, src_stride, dst, dst_stride,
256 filter_x, w, h, bd);
257 } else {
258 highbd_convolve_horiz_8tap_neon(src, src_stride, dst, dst_stride,
259 filter_x, w, h, bd);
260 }
261 }
262 }
263
highbd_convolve_vert_8tap_neon(const uint16_t * src_ptr,ptrdiff_t src_stride,uint16_t * dst_ptr,ptrdiff_t dst_stride,const int16_t * y_filter_ptr,int w,int h,int bd)264 static void highbd_convolve_vert_8tap_neon(
265 const uint16_t *src_ptr, ptrdiff_t src_stride, uint16_t *dst_ptr,
266 ptrdiff_t dst_stride, const int16_t *y_filter_ptr, int w, int h, int bd) {
267 assert(w >= 4 && h >= 4);
268 const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
269
270 if (w == 4) {
271 const uint16x4_t max = vdup_n_u16((1 << bd) - 1);
272 const int16_t *s = (const int16_t *)src_ptr;
273 uint16_t *d = dst_ptr;
274
275 int16x4_t s0, s1, s2, s3, s4, s5, s6;
276 load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
277 s += 7 * src_stride;
278
279 do {
280 int16x4_t s7, s8, s9, s10;
281 load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
282
283 uint16x4_t d0 =
284 highbd_convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, max);
285 uint16x4_t d1 =
286 highbd_convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, max);
287 uint16x4_t d2 =
288 highbd_convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, max);
289 uint16x4_t d3 =
290 highbd_convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, max);
291
292 store_u16_4x4(d, dst_stride, d0, d1, d2, d3);
293
294 s0 = s4;
295 s1 = s5;
296 s2 = s6;
297 s3 = s7;
298 s4 = s8;
299 s5 = s9;
300 s6 = s10;
301
302 s += 4 * src_stride;
303 d += 4 * dst_stride;
304 h -= 4;
305 } while (h > 0);
306 } else {
307 const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
308
309 do {
310 int height = h;
311 const int16_t *s = (const int16_t *)src_ptr;
312 uint16_t *d = dst_ptr;
313
314 int16x8_t s0, s1, s2, s3, s4, s5, s6;
315 load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
316 s += 7 * src_stride;
317
318 do {
319 int16x8_t s7, s8, s9, s10;
320 load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
321
322 uint16x8_t d0 =
323 highbd_convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, max);
324 uint16x8_t d1 =
325 highbd_convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, max);
326 uint16x8_t d2 =
327 highbd_convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, max);
328 uint16x8_t d3 =
329 highbd_convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, max);
330
331 store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
332
333 s0 = s4;
334 s1 = s5;
335 s2 = s6;
336 s3 = s7;
337 s4 = s8;
338 s5 = s9;
339 s6 = s10;
340
341 s += 4 * src_stride;
342 d += 4 * dst_stride;
343 height -= 4;
344 } while (height > 0);
345 src_ptr += 8;
346 dst_ptr += 8;
347 w -= 8;
348 } while (w > 0);
349 }
350 }
351
aom_highbd_convolve8_vert_neon(const uint8_t * src8,ptrdiff_t src_stride,uint8_t * dst8,ptrdiff_t dst_stride,const int16_t * filter_x,int x_step_q4,const int16_t * filter_y,int y_step_q4,int w,int h,int bd)352 void aom_highbd_convolve8_vert_neon(const uint8_t *src8, ptrdiff_t src_stride,
353 uint8_t *dst8, ptrdiff_t dst_stride,
354 const int16_t *filter_x, int x_step_q4,
355 const int16_t *filter_y, int y_step_q4,
356 int w, int h, int bd) {
357 if (y_step_q4 != 16) {
358 aom_highbd_convolve8_vert_c(src8, src_stride, dst8, dst_stride, filter_x,
359 x_step_q4, filter_y, y_step_q4, w, h, bd);
360 } else {
361 (void)filter_x;
362 (void)x_step_q4;
363
364 uint16_t *src = CONVERT_TO_SHORTPTR(src8);
365 uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
366
367 src -= (SUBPEL_TAPS / 2 - 1) * src_stride;
368
369 const int filter_taps = get_filter_taps_convolve8(filter_y);
370
371 if (filter_taps == 2) {
372 highbd_convolve8_vert_2tap_neon(src + 3 * src_stride, src_stride, dst,
373 dst_stride, filter_y, w, h, bd);
374 } else if (filter_taps == 4) {
375 highbd_convolve8_vert_4tap_neon(src + 2 * src_stride, src_stride, dst,
376 dst_stride, filter_y, w, h, bd);
377 } else {
378 highbd_convolve_vert_8tap_neon(src, src_stride, dst, dst_stride, filter_y,
379 w, h, bd);
380 }
381 }
382 }
383