1 /*
2 * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12 #include <assert.h>
13 #include <string.h>
14
15 #include "./vpx_config.h"
16 #include "./vpx_dsp_rtcd.h"
17 #include "vpx/vpx_integer.h"
18 #include "vpx_dsp/arm/mem_neon.h"
19 #include "vpx_dsp/arm/transpose_neon.h"
20 #include "vpx_dsp/arm/vpx_convolve8_neon.h"
21 #include "vpx_dsp/vpx_filter.h"
22 #include "vpx_ports/mem.h"
23
scaledconvolve_horiz_neon(const uint8_t * src,const ptrdiff_t src_stride,uint8_t * dst,const ptrdiff_t dst_stride,const InterpKernel * const x_filter,const int x0_q4,const int x_step_q4,int w,int h)24 static INLINE void scaledconvolve_horiz_neon(
25 const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
26 const ptrdiff_t dst_stride, const InterpKernel *const x_filter,
27 const int x0_q4, const int x_step_q4, int w, int h) {
28 DECLARE_ALIGNED(16, uint8_t, temp[8 * 8]);
29
30 src -= SUBPEL_TAPS / 2 - 1;
31
32 if (w == 4) {
33 do {
34 int x_q4 = x0_q4;
35
36 // Process a 4x4 tile.
37 for (int r = 0; r < 4; ++r) {
38 const uint8_t *s = &src[x_q4 >> SUBPEL_BITS];
39
40 if (x_q4 & SUBPEL_MASK) {
41 const int16x8_t filter = vld1q_s16(x_filter[x_q4 & SUBPEL_MASK]);
42
43 uint8x8_t t0, t1, t2, t3;
44 load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
45 transpose_u8_8x4(&t0, &t1, &t2, &t3);
46
47 int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
48 int16x4_t s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
49 int16x4_t s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
50 int16x4_t s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
51 int16x4_t s4 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
52 int16x4_t s5 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
53 int16x4_t s6 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
54 int16x4_t s7 = vget_high_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
55
56 int16x4_t dd0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filter);
57 uint8x8_t d0 =
58 vqrshrun_n_s16(vcombine_s16(dd0, vdup_n_s16(0)), FILTER_BITS);
59
60 store_u8_4x1(&temp[4 * r], d0);
61 } else {
62 // Memcpy for non-subpel locations.
63 s += SUBPEL_TAPS / 2 - 1;
64
65 for (int c = 0; c < 4; ++c) {
66 temp[r * 4 + c] = s[c * src_stride];
67 }
68 }
69 x_q4 += x_step_q4;
70 }
71
72 // Transpose the 4x4 result tile and store.
73 uint8x8_t d01 = vld1_u8(temp + 0);
74 uint8x8_t d23 = vld1_u8(temp + 8);
75
76 transpose_u8_4x4(&d01, &d23);
77
78 store_u8_4x1(dst + 0 * dst_stride, d01);
79 store_u8_4x1(dst + 1 * dst_stride, d23);
80 store_u8_4x1_high(dst + 2 * dst_stride, d01);
81 store_u8_4x1_high(dst + 3 * dst_stride, d23);
82
83 src += 4 * src_stride;
84 dst += 4 * dst_stride;
85 h -= 4;
86 } while (h > 0);
87 return;
88 }
89
90 do {
91 int x_q4 = x0_q4;
92 uint8_t *d = dst;
93 int width = w;
94
95 do {
96 // Process an 8x8 tile.
97 for (int r = 0; r < 8; ++r) {
98 const uint8_t *s = &src[x_q4 >> SUBPEL_BITS];
99
100 if (x_q4 & SUBPEL_MASK) {
101 const int16x8_t filter = vld1q_s16(x_filter[x_q4 & SUBPEL_MASK]);
102
103 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
104 load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
105
106 transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
107 int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
108 int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
109 int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
110 int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
111 int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
112 int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
113 int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
114 int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
115
116 uint8x8_t d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filter);
117
118 vst1_u8(&temp[r * 8], d0);
119 } else {
120 // Memcpy for non-subpel locations.
121 s += SUBPEL_TAPS / 2 - 1;
122
123 for (int c = 0; c < 8; ++c) {
124 temp[r * 8 + c] = s[c * src_stride];
125 }
126 }
127 x_q4 += x_step_q4;
128 }
129
130 // Transpose the 8x8 result tile and store.
131 uint8x8_t d0, d1, d2, d3, d4, d5, d6, d7;
132 load_u8_8x8(temp, 8, &d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
133
134 transpose_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
135
136 store_u8_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
137
138 d += 8;
139 width -= 8;
140 } while (width != 0);
141
142 src += 8 * src_stride;
143 dst += 8 * dst_stride;
144 h -= 8;
145 } while (h > 0);
146 }
147
scaledconvolve_vert_neon(const uint8_t * src,const ptrdiff_t src_stride,uint8_t * dst,const ptrdiff_t dst_stride,const InterpKernel * const y_filter,const int y0_q4,const int y_step_q4,int w,int h)148 static INLINE void scaledconvolve_vert_neon(
149 const uint8_t *src, const ptrdiff_t src_stride, uint8_t *dst,
150 const ptrdiff_t dst_stride, const InterpKernel *const y_filter,
151 const int y0_q4, const int y_step_q4, int w, int h) {
152 int y_q4 = y0_q4;
153
154 if (w == 4) {
155 do {
156 const uint8_t *s = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
157
158 if (y_q4 & SUBPEL_MASK) {
159 const int16x8_t filter = vld1q_s16(y_filter[y_q4 & SUBPEL_MASK]);
160
161 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
162 load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
163 int16x4_t s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
164 int16x4_t s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
165 int16x4_t s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
166 int16x4_t s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
167 int16x4_t s4 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t4)));
168 int16x4_t s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
169 int16x4_t s6 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t6)));
170 int16x4_t s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t7)));
171
172 int16x4_t dd0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filter);
173 uint8x8_t d0 =
174 vqrshrun_n_s16(vcombine_s16(dd0, vdup_n_s16(0)), FILTER_BITS);
175
176 store_u8_4x1(dst, d0);
177 } else {
178 // Memcpy for non-subpel locations.
179 memcpy(dst, &s[(SUBPEL_TAPS / 2 - 1) * src_stride], 4);
180 }
181
182 y_q4 += y_step_q4;
183 dst += dst_stride;
184 } while (--h != 0);
185 return;
186 }
187
188 if (w == 8) {
189 do {
190 const uint8_t *s = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
191
192 if (y_q4 & SUBPEL_MASK) {
193 const int16x8_t filter = vld1q_s16(y_filter[y_q4 & SUBPEL_MASK]);
194
195 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
196 load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
197 int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
198 int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
199 int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
200 int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
201 int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
202 int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
203 int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
204 int16x8_t s7 = vreinterpretq_s16_u16(vmovl_u8(t7));
205
206 uint8x8_t d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filter);
207
208 vst1_u8(dst, d0);
209 } else {
210 // Memcpy for non-subpel locations.
211 memcpy(dst, &s[(SUBPEL_TAPS / 2 - 1) * src_stride], 8);
212 }
213
214 y_q4 += y_step_q4;
215 dst += dst_stride;
216 } while (--h != 0);
217 return;
218 }
219
220 do {
221 const uint8_t *s = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
222 uint8_t *d = dst;
223 int width = w;
224
225 if (y_q4 & SUBPEL_MASK) {
226 do {
227 const int16x8_t filter = vld1q_s16(y_filter[y_q4 & SUBPEL_MASK]);
228
229 uint8x16_t t0, t1, t2, t3, t4, t5, t6, t7;
230 load_u8_16x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
231
232 int16x8_t s0[2], s1[2], s2[2], s3[2], s4[2], s5[2], s6[2], s7[2];
233 s0[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t0)));
234 s1[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t1)));
235 s2[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t2)));
236 s3[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t3)));
237 s4[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t4)));
238 s5[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t5)));
239 s6[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t6)));
240 s7[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(t7)));
241
242 s0[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t0)));
243 s1[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t1)));
244 s2[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t2)));
245 s3[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t3)));
246 s4[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t4)));
247 s5[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t5)));
248 s6[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t6)));
249 s7[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(t7)));
250
251 uint8x8_t d0 = convolve8_8(s0[0], s1[0], s2[0], s3[0], s4[0], s5[0],
252 s6[0], s7[0], filter);
253 uint8x8_t d1 = convolve8_8(s0[1], s1[1], s2[1], s3[1], s4[1], s5[1],
254 s6[1], s7[1], filter);
255
256 vst1q_u8(d, vcombine_u8(d0, d1));
257
258 s += 16;
259 d += 16;
260 width -= 16;
261 } while (width != 0);
262 } else {
263 // Memcpy for non-subpel locations.
264 s += (SUBPEL_TAPS / 2 - 1) * src_stride;
265
266 do {
267 uint8x16_t s0 = vld1q_u8(s);
268 vst1q_u8(d, s0);
269 s += 16;
270 d += 16;
271 width -= 16;
272 } while (width != 0);
273 }
274
275 y_q4 += y_step_q4;
276 dst += dst_stride;
277 } while (--h != 0);
278 }
279
vpx_scaled_2d_neon(const uint8_t * src,ptrdiff_t src_stride,uint8_t * dst,ptrdiff_t dst_stride,const InterpKernel * filter,int x0_q4,int x_step_q4,int y0_q4,int y_step_q4,int w,int h)280 void vpx_scaled_2d_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
281 ptrdiff_t dst_stride, const InterpKernel *filter,
282 int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
283 int w, int h) {
284 // Fixed size intermediate buffer, im_block, places limits on parameters.
285 // 2d filtering proceeds in 2 steps:
286 // (1) Interpolate horizontally into an intermediate buffer, temp.
287 // (2) Interpolate temp vertically to derive the sub-pixel result.
288 // Deriving the maximum number of rows in the im_block buffer (135):
289 // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
290 // --Largest block size is 64x64 pixels.
291 // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
292 // original frame (in 1/16th pixel units).
293 // --Must round-up because block may be located at sub-pixel position.
294 // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
295 // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
296 // --Require an additional 8 rows for the horiz_w8 transpose tail.
297 // When calling in frame scaling function, the smallest scaling factor is x1/4
298 // ==> y_step_q4 = 64. Since w and h are at most 16, the temp buffer is still
299 // big enough.
300 DECLARE_ALIGNED(16, uint8_t, im_block[(135 + 8) * 64]);
301 const int im_height =
302 (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
303 const ptrdiff_t im_stride = 64;
304
305 assert(w <= 64);
306 assert(h <= 64);
307 assert(y_step_q4 <= 32 || (y_step_q4 <= 64 && h <= 32));
308 assert(x_step_q4 <= 64);
309
310 scaledconvolve_horiz_neon(src - src_stride * (SUBPEL_TAPS / 2 - 1),
311 src_stride, im_block, im_stride, filter, x0_q4,
312 x_step_q4, w, im_height);
313
314 scaledconvolve_vert_neon(im_block, im_stride, dst, dst_stride, filter, y0_q4,
315 y_step_q4, w, h);
316 }
317