1 /*
2 * Copyright (c) 2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "arm_compute/core/Helpers.h"
26
27 #include <arm_neon.h>
28 #include <cstdint>
29
30 namespace arm_compute
31 {
32 namespace cpu
33 {
34 namespace
35 {
36 #ifdef __aarch64__
37
substitute_bytes_neon(const uint8_t * table,size_t num_strings,size_t string_length,const uint8_t * const * input,uint8_t * const * output)38 void substitute_bytes_neon(
39 const uint8_t *table,
40 size_t num_strings,
41 size_t string_length,
42 const uint8_t *const *input,
43 uint8_t *const *output)
44 {
45 __asm__ __volatile__(
46 "ldr q16, [%x[table], #0x0]\n"
47 "ldr q17, [%x[table], #0x10]\n"
48 "mov x22, #0x0\n"
49 "ldr q18, [%x[table], #0x20]\n"
50 "ldr q19, [%x[table], #0x30]\n"
51 "ldr q20, [%x[table], #0x40]\n"
52 "ldr q21, [%x[table], #0x50]\n"
53 "ldr q22, [%x[table], #0x60]\n"
54 "ldr q23, [%x[table], #0x70]\n"
55 "ldr q24, [%x[table], #0x80]\n"
56 "ldr q25, [%x[table], #0x90]\n"
57 "ldr q26, [%x[table], #0xa0]\n"
58 "ldr q27, [%x[table], #0xb0]\n"
59 "ldr q28, [%x[table], #0xc0]\n"
60 "ldr q29, [%x[table], #0xd0]\n"
61 "ldr q30, [%x[table], #0xe0]\n"
62 "ldr q31, [%x[table], #0xf0]\n"
63 "1:" // string loop
64 "ldr x21, [%x[input], x22, LSL #0x3]\n"
65 "ldr x20, [%x[output], x22, LSL #0x3]\n"
66 "movi v12.16b, #0x40\n"
67 "movi v11.16b, #0x80\n"
68 "movi v10.16b, #0xc0\n"
69 "mov x19, %x[string_length]\n"
70 "2:" // 4 rounds: width loop
71 "cmp x19, #0x30\n"
72 "bge 27f\n"
73 "tbz x19, #5, 10f\n"
74 "ld1 { v9.16b }, [x21], #0x10\n"
75 "ld1 { v13.16b }, [x21], #0x10\n"
76 "tbz x19, #3, 6f\n"
77 "ldr d14, [x21], #0x8\n"
78 "tbz x19, #2, 4f\n"
79 "ld1 { v14.s }[2], [x21], #0x4\n"
80 "tbz x19, #1, 3f\n"
81 "ld1 { v14.h }[6], [x21], #0x2\n"
82 "tbz x19, #0, 26f\n"
83 "ld1 { v14.b }[14], [x21]\n"
84 "b 26f\n"
85 "3:" // 4 rounds: Partial load: partial_1_44
86 "tbz x19, #0, 26f\n"
87 "ld1 { v14.b }[12], [x21]\n"
88 "b 26f\n"
89 "4:" // 4 rounds: Partial load: partial_2_40
90 "tbz x19, #1, 5f\n"
91 "ld1 { v14.h }[4], [x21], #0x2\n"
92 "tbz x19, #0, 26f\n"
93 "ld1 { v14.b }[10], [x21]\n"
94 "b 26f\n"
95 "5:" // 4 rounds: Partial load: partial_1_40
96 "tbz x19, #0, 26f\n"
97 "ld1 { v14.b }[8], [x21]\n"
98 "b 26f\n"
99 "6:" // 4 rounds: Partial load: partial_4_32
100 "tbz x19, #2, 8f\n"
101 "ldr s14, [x21], #0x4\n"
102 "tbz x19, #1, 7f\n"
103 "ld1 { v14.h }[2], [x21], #0x2\n"
104 "tbz x19, #0, 26f\n"
105 "ld1 { v14.b }[6], [x21]\n"
106 "b 26f\n"
107 "7:" // 4 rounds: Partial load: partial_1_36
108 "tbz x19, #0, 26f\n"
109 "ld1 { v14.b }[4], [x21]\n"
110 "b 26f\n"
111 "8:" // 4 rounds: Partial load: partial_2_32
112 "tbz x19, #1, 9f\n"
113 "ldr h14, [x21], #0x2\n"
114 "tbz x19, #0, 26f\n"
115 "ld1 { v14.b }[2], [x21]\n"
116 "b 26f\n"
117 "9:" // 4 rounds: Partial load: partial_1_32
118 "tbz x19, #0, 26f\n"
119 "ldr b14, [x21, #0x0]\n"
120 "b 26f\n"
121 "10:" // 4 rounds: Partial load: partial_16_0
122 "tbz x19, #4, 18f\n"
123 "ld1 { v9.16b }, [x21], #0x10\n"
124 "tbz x19, #3, 14f\n"
125 "ldr d13, [x21], #0x8\n"
126 "tbz x19, #2, 12f\n"
127 "ld1 { v13.s }[2], [x21], #0x4\n"
128 "tbz x19, #1, 11f\n"
129 "ld1 { v13.h }[6], [x21], #0x2\n"
130 "tbz x19, #0, 26f\n"
131 "ld1 { v13.b }[14], [x21]\n"
132 "b 26f\n"
133 "11:" // 4 rounds: Partial load: partial_1_28
134 "tbz x19, #0, 26f\n"
135 "ld1 { v13.b }[12], [x21]\n"
136 "b 26f\n"
137 "12:" // 4 rounds: Partial load: partial_2_24
138 "tbz x19, #1, 13f\n"
139 "ld1 { v13.h }[4], [x21], #0x2\n"
140 "tbz x19, #0, 26f\n"
141 "ld1 { v13.b }[10], [x21]\n"
142 "b 26f\n"
143 "13:" // 4 rounds: Partial load: partial_1_24
144 "tbz x19, #0, 26f\n"
145 "ld1 { v13.b }[8], [x21]\n"
146 "b 26f\n"
147 "14:" // 4 rounds: Partial load: partial_4_16
148 "tbz x19, #2, 16f\n"
149 "ldr s13, [x21], #0x4\n"
150 "tbz x19, #1, 15f\n"
151 "ld1 { v13.h }[2], [x21], #0x2\n"
152 "tbz x19, #0, 26f\n"
153 "ld1 { v13.b }[6], [x21]\n"
154 "b 26f\n"
155 "15:" // 4 rounds: Partial load: partial_1_20
156 "tbz x19, #0, 26f\n"
157 "ld1 { v13.b }[4], [x21]\n"
158 "b 26f\n"
159 "16:" // 4 rounds: Partial load: partial_2_16
160 "tbz x19, #1, 17f\n"
161 "ldr h13, [x21], #0x2\n"
162 "tbz x19, #0, 26f\n"
163 "ld1 { v13.b }[2], [x21]\n"
164 "b 26f\n"
165 "17:" // 4 rounds: Partial load: partial_1_16
166 "tbz x19, #0, 26f\n"
167 "ldr b13, [x21, #0x0]\n"
168 "b 26f\n"
169 "18:" // 4 rounds: Partial load: partial_8_0
170 "tbz x19, #3, 22f\n"
171 "ldr d9, [x21], #0x8\n"
172 "tbz x19, #2, 20f\n"
173 "ld1 { v9.s }[2], [x21], #0x4\n"
174 "tbz x19, #1, 19f\n"
175 "ld1 { v9.h }[6], [x21], #0x2\n"
176 "tbz x19, #0, 26f\n"
177 "ld1 { v9.b }[14], [x21]\n"
178 "b 26f\n"
179 "19:" // 4 rounds: Partial load: partial_1_12
180 "tbz x19, #0, 26f\n"
181 "ld1 { v9.b }[12], [x21]\n"
182 "b 26f\n"
183 "20:" // 4 rounds: Partial load: partial_2_8
184 "tbz x19, #1, 21f\n"
185 "ld1 { v9.h }[4], [x21], #0x2\n"
186 "tbz x19, #0, 26f\n"
187 "ld1 { v9.b }[10], [x21]\n"
188 "b 26f\n"
189 "21:" // 4 rounds: Partial load: partial_1_8
190 "tbz x19, #0, 26f\n"
191 "ld1 { v9.b }[8], [x21]\n"
192 "b 26f\n"
193 "22:" // 4 rounds: Partial load: partial_4_0
194 "tbz x19, #2, 24f\n"
195 "ldr s9, [x21], #0x4\n"
196 "tbz x19, #1, 23f\n"
197 "ld1 { v9.h }[2], [x21], #0x2\n"
198 "tbz x19, #0, 26f\n"
199 "ld1 { v9.b }[6], [x21]\n"
200 "b 26f\n"
201 "23:" // 4 rounds: Partial load: partial_1_4
202 "tbz x19, #0, 26f\n"
203 "ld1 { v9.b }[4], [x21]\n"
204 "b 26f\n"
205 "24:" // 4 rounds: Partial load: partial_2_0
206 "tbz x19, #1, 25f\n"
207 "ldr h9, [x21], #0x2\n"
208 "tbz x19, #0, 26f\n"
209 "ld1 { v9.b }[2], [x21]\n"
210 "b 26f\n"
211 "25:" // 4 rounds: Partial load: partial_1_0
212 "ldr b9, [x21, #0x0]\n"
213 "26:" // 4 rounds: Partial load: Done
214 "b 28f\n"
215 "27:" // 4 rounds: Full load
216 "ldr q9, [x21, #0x0]\n"
217 "ldr q13, [x21, #0x10]\n"
218 "ldr q14, [x21, #0x20]\n"
219 "add x21, x21, #0x30\n"
220 "28:" // 4 rounds: Load done
221 "sub v8.16b, v9.16b, v12.16b\n"
222 "sub v7.16b, v9.16b, v11.16b\n"
223 "tbl v8.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v8.16b\n"
224 "sub v6.16b, v9.16b, v10.16b\n"
225 "sub v5.16b, v13.16b, v12.16b\n"
226 "tbl v9.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v9.16b\n"
227 "sub v4.16b, v13.16b, v11.16b\n"
228 "sub v3.16b, v13.16b, v10.16b\n"
229 "tbl v7.16b, { v24.16b, v25.16b, v26.16b, v27.16b }, v7.16b\n"
230 "sub v2.16b, v14.16b, v12.16b\n"
231 "sub v1.16b, v14.16b, v11.16b\n"
232 "tbl v6.16b, { v28.16b, v29.16b, v30.16b, v31.16b }, v6.16b\n"
233 "sub v0.16b, v14.16b, v10.16b\n"
234 "tbl v13.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v13.16b\n"
235 "tbl v5.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v5.16b\n"
236 "tbl v4.16b, { v24.16b, v25.16b, v26.16b, v27.16b }, v4.16b\n"
237 "tbl v3.16b, { v28.16b, v29.16b, v30.16b, v31.16b }, v3.16b\n"
238 "orr v9.16b, v9.16b, v8.16b\n"
239 "tbl v14.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v14.16b\n"
240 "tbl v2.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v2.16b\n"
241 "orr v7.16b, v7.16b, v6.16b\n"
242 "tbl v1.16b, { v24.16b, v25.16b, v26.16b, v27.16b }, v1.16b\n"
243 "tbl v0.16b, { v28.16b, v29.16b, v30.16b, v31.16b }, v0.16b\n"
244 "orr v13.16b, v13.16b, v5.16b\n"
245 "orr v4.16b, v4.16b, v3.16b\n"
246 "orr v14.16b, v14.16b, v2.16b\n"
247 "cmp x19, #0x30\n"
248 "orr v1.16b, v1.16b, v0.16b\n"
249 "orr v9.16b, v9.16b, v7.16b\n"
250 "orr v13.16b, v13.16b, v4.16b\n"
251 "orr v14.16b, v14.16b, v1.16b\n"
252 "bge 53f\n"
253 "tbz x19, #5, 36f\n"
254 "st1 { v9.16b }, [x20], #0x10\n"
255 "st1 { v13.16b }, [x20], #0x10\n"
256 "tbz x19, #3, 32f\n"
257 "str d14, [x20], #0x8\n"
258 "tbz x19, #2, 30f\n"
259 "st1 { v14.s }[2], [x20], #0x4\n"
260 "tbz x19, #1, 29f\n"
261 "st1 { v14.h }[6], [x20], #0x2\n"
262 "tbz x19, #0, 52f\n"
263 "st1 { v14.b }[14], [x20]\n"
264 "b 52f\n"
265 "29:" // 4 rounds: Partial writeback: partial_1_44
266 "tbz x19, #0, 52f\n"
267 "st1 { v14.b }[12], [x20]\n"
268 "b 52f\n"
269 "30:" // 4 rounds: Partial writeback: partial_2_40
270 "tbz x19, #1, 31f\n"
271 "st1 { v14.h }[4], [x20], #0x2\n"
272 "tbz x19, #0, 52f\n"
273 "st1 { v14.b }[10], [x20]\n"
274 "b 52f\n"
275 "31:" // 4 rounds: Partial writeback: partial_1_40
276 "tbz x19, #0, 52f\n"
277 "st1 { v14.b }[8], [x20]\n"
278 "b 52f\n"
279 "32:" // 4 rounds: Partial writeback: partial_4_32
280 "tbz x19, #2, 34f\n"
281 "str s14, [x20], #0x4\n"
282 "tbz x19, #1, 33f\n"
283 "st1 { v14.h }[2], [x20], #0x2\n"
284 "tbz x19, #0, 52f\n"
285 "st1 { v14.b }[6], [x20]\n"
286 "b 52f\n"
287 "33:" // 4 rounds: Partial writeback: partial_1_36
288 "tbz x19, #0, 52f\n"
289 "st1 { v14.b }[4], [x20]\n"
290 "b 52f\n"
291 "34:" // 4 rounds: Partial writeback: partial_2_32
292 "tbz x19, #1, 35f\n"
293 "str h14, [x20], #0x2\n"
294 "tbz x19, #0, 52f\n"
295 "st1 { v14.b }[2], [x20]\n"
296 "b 52f\n"
297 "35:" // 4 rounds: Partial writeback: partial_1_32
298 "tbz x19, #0, 52f\n"
299 "str b14, [x20, #0x0]\n"
300 "b 52f\n"
301 "36:" // 4 rounds: Partial writeback: partial_16_0
302 "tbz x19, #4, 44f\n"
303 "st1 { v9.16b }, [x20], #0x10\n"
304 "tbz x19, #3, 40f\n"
305 "str d13, [x20], #0x8\n"
306 "tbz x19, #2, 38f\n"
307 "st1 { v13.s }[2], [x20], #0x4\n"
308 "tbz x19, #1, 37f\n"
309 "st1 { v13.h }[6], [x20], #0x2\n"
310 "tbz x19, #0, 52f\n"
311 "st1 { v13.b }[14], [x20]\n"
312 "b 52f\n"
313 "37:" // 4 rounds: Partial writeback: partial_1_28
314 "tbz x19, #0, 52f\n"
315 "st1 { v13.b }[12], [x20]\n"
316 "b 52f\n"
317 "38:" // 4 rounds: Partial writeback: partial_2_24
318 "tbz x19, #1, 39f\n"
319 "st1 { v13.h }[4], [x20], #0x2\n"
320 "tbz x19, #0, 52f\n"
321 "st1 { v13.b }[10], [x20]\n"
322 "b 52f\n"
323 "39:" // 4 rounds: Partial writeback: partial_1_24
324 "tbz x19, #0, 52f\n"
325 "st1 { v13.b }[8], [x20]\n"
326 "b 52f\n"
327 "40:" // 4 rounds: Partial writeback: partial_4_16
328 "tbz x19, #2, 42f\n"
329 "str s13, [x20], #0x4\n"
330 "tbz x19, #1, 41f\n"
331 "st1 { v13.h }[2], [x20], #0x2\n"
332 "tbz x19, #0, 52f\n"
333 "st1 { v13.b }[6], [x20]\n"
334 "b 52f\n"
335 "41:" // 4 rounds: Partial writeback: partial_1_20
336 "tbz x19, #0, 52f\n"
337 "st1 { v13.b }[4], [x20]\n"
338 "b 52f\n"
339 "42:" // 4 rounds: Partial writeback: partial_2_16
340 "tbz x19, #1, 43f\n"
341 "str h13, [x20], #0x2\n"
342 "tbz x19, #0, 52f\n"
343 "st1 { v13.b }[2], [x20]\n"
344 "b 52f\n"
345 "43:" // 4 rounds: Partial writeback: partial_1_16
346 "tbz x19, #0, 52f\n"
347 "str b13, [x20, #0x0]\n"
348 "b 52f\n"
349 "44:" // 4 rounds: Partial writeback: partial_8_0
350 "tbz x19, #3, 48f\n"
351 "str d9, [x20], #0x8\n"
352 "tbz x19, #2, 46f\n"
353 "st1 { v9.s }[2], [x20], #0x4\n"
354 "tbz x19, #1, 45f\n"
355 "st1 { v9.h }[6], [x20], #0x2\n"
356 "tbz x19, #0, 52f\n"
357 "st1 { v9.b }[14], [x20]\n"
358 "b 52f\n"
359 "45:" // 4 rounds: Partial writeback: partial_1_12
360 "tbz x19, #0, 52f\n"
361 "st1 { v9.b }[12], [x20]\n"
362 "b 52f\n"
363 "46:" // 4 rounds: Partial writeback: partial_2_8
364 "tbz x19, #1, 47f\n"
365 "st1 { v9.h }[4], [x20], #0x2\n"
366 "tbz x19, #0, 52f\n"
367 "st1 { v9.b }[10], [x20]\n"
368 "b 52f\n"
369 "47:" // 4 rounds: Partial writeback: partial_1_8
370 "tbz x19, #0, 52f\n"
371 "st1 { v9.b }[8], [x20]\n"
372 "b 52f\n"
373 "48:" // 4 rounds: Partial writeback: partial_4_0
374 "tbz x19, #2, 50f\n"
375 "str s9, [x20], #0x4\n"
376 "tbz x19, #1, 49f\n"
377 "st1 { v9.h }[2], [x20], #0x2\n"
378 "tbz x19, #0, 52f\n"
379 "st1 { v9.b }[6], [x20]\n"
380 "b 52f\n"
381 "49:" // 4 rounds: Partial writeback: partial_1_4
382 "tbz x19, #0, 52f\n"
383 "st1 { v9.b }[4], [x20]\n"
384 "b 52f\n"
385 "50:" // 4 rounds: Partial writeback: partial_2_0
386 "tbz x19, #1, 51f\n"
387 "str h9, [x20], #0x2\n"
388 "tbz x19, #0, 52f\n"
389 "st1 { v9.b }[2], [x20]\n"
390 "b 52f\n"
391 "51:" // 4 rounds: Partial writeback: partial_1_0
392 "str b9, [x20, #0x0]\n"
393 "52:" // 4 rounds: Partial writeback: Done
394 "b 54f\n"
395 "53:" // 4 rounds: Full writeback
396 "str q9, [x20, #0x0]\n"
397 "str q13, [x20, #0x10]\n"
398 "str q14, [x20, #0x20]\n"
399 "add x20, x20, #0x30\n"
400 "54:" // 4 rounds: Writeback done
401 "subs x19, x19, #0x30\n"
402 "bgt 2b\n"
403 "add x22, x22, #0x1\n"
404 "cmp x22, %x[num_strings]\n"
405 "bne 1b\n"
406 :
407 : [input] "r"(input), [num_strings] "r"(num_strings), [output] "r"(output), [string_length] "r"(string_length), [table] "r"(table)
408 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22");
409 }
410
411 #endif // __aarch64__
412 } // namespace
413
414 #ifdef __aarch64__
neon_q8_activation_lut(const ITensor * src,ITensor * dst,const ActivationLayerInfo & act_info,const Window & window)415 void neon_q8_activation_lut(const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window)
416 {
417 ARM_COMPUTE_ERROR_ON(!ActivationLayerInfo::is_lut_supported(act_info.activation(), src->info()->data_type()));
418 const auto window_end_x = window.x().end();
419 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
420 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
421 Iterator input(src, win_collapsed);
422 Iterator output(dst, win_collapsed);
423 execute_window_loop(win_collapsed, [&](const Coordinates &)
424 {
425 const auto input_ptr = reinterpret_cast<const uint8_t *>(input.ptr());
426 auto output_ptr = reinterpret_cast<uint8_t *>(output.ptr());
427 substitute_bytes_neon(act_info.lut().data(), 1u, window_end_x, &input_ptr, &output_ptr);
428 },
429 input, output);
430 }
431 #endif // __aarch64__
432 } // namespace cpu
433 } // namespace arm_compute
434