1 /*
2 * Copyright (c) 2019-2021 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #ifdef __aarch64__
26
27 template<>
interleave_block(int8_t * & out_ptr,const int8_t * const * in,size_t width,size_t height,size_t row_offset,bool)28 void interleave_block<4, 16, VLType::None, false>(
29 int8_t * &out_ptr, const int8_t * const * in, size_t width, size_t height,
30 size_t row_offset, bool
31 )
32 {
33 __asm__ __volatile__(
34 "ldr x22, [%x[in], #0x0]\n"
35 "cmp %x[height], #0x4\n"
36 "ldr x21, [%x[in], #0x8]\n"
37 "add x22, x22, %x[row_offset]\n"
38 "ldr x20, [%x[in], #0x10]\n"
39 "ldr x19, [%x[in], #0x18]\n"
40 "add x21, x21, %x[row_offset]\n"
41 "add x20, x20, %x[row_offset]\n"
42 "add x19, x19, %x[row_offset]\n"
43 "beq 1f\n"
44 "mov x19, x22\n"
45 "cmp %x[height], #0x2\n"
46 "csel x21, x21, x22, GE\n"
47 "csel x20, x20, x22, GT\n"
48 "1:" // no_pointer_adj
49 "prfm pldl1keep, [x22, #0x0]\n"
50 "cmp %x[width], #0x10\n"
51 "prfm pldl1keep, [x21, #0x0]\n"
52 "prfm pldl1keep, [x20, #0x0]\n"
53 "prfm pldl1keep, [x19, #0x0]\n"
54 "prfm pldl1keep, [x22, #0x40]\n"
55 "prfm pldl1keep, [x21, #0x40]\n"
56 "prfm pldl1keep, [x20, #0x40]\n"
57 "prfm pldl1keep, [x19, #0x40]\n"
58 "blt 3f\n"
59 "2:" // Main loop head
60 "ldr q19, [x22], #0x10\n"
61 "subs %x[width], %x[width], #0x10\n"
62 "ldr q18, [x21], #0x10\n"
63 "cmp %x[width], #0x10\n"
64 "ldr q17, [x20], #0x10\n"
65 "ldr q16, [x19], #0x10\n"
66 "prfm pldl1keep, [x22, #0x70]\n"
67 "prfm pldl1keep, [x21, #0x70]\n"
68 "prfm pldl1keep, [x20, #0x70]\n"
69 "prfm pldl1keep, [x19, #0x70]\n"
70 "str q19, [%x[out_ptr], #0x0]\n"
71 "str q18, [%x[out_ptr], #0x10]\n"
72 "str q17, [%x[out_ptr], #0x20]\n"
73 "str q16, [%x[out_ptr], #0x30]\n"
74 "add %x[out_ptr], %x[out_ptr], #0x40\n"
75 "bge 2b\n"
76 "3:" // Main loop skip
77 "cbz %x[width], 12f\n"
78 "tbz %x[width], #3, 7f\n"
79 "ldr d19, [x22], #0x8\n"
80 "ldr d18, [x21], #0x8\n"
81 "ldr d17, [x20], #0x8\n"
82 "ldr d16, [x19], #0x8\n"
83 "tbz %x[width], #2, 5f\n"
84 "ld1 { v19.s }[2], [x22], #0x4\n"
85 "ld1 { v18.s }[2], [x21], #0x4\n"
86 "ld1 { v17.s }[2], [x20], #0x4\n"
87 "ld1 { v16.s }[2], [x19], #0x4\n"
88 "tbz %x[width], #1, 4f\n"
89 "ld1 { v19.h }[6], [x22], #0x2\n"
90 "ld1 { v18.h }[6], [x21], #0x2\n"
91 "ld1 { v17.h }[6], [x20], #0x2\n"
92 "ld1 { v16.h }[6], [x19], #0x2\n"
93 "tbz %x[width], #0, 11f\n"
94 "ld1 { v19.b }[14], [x22]\n"
95 "ld1 { v18.b }[14], [x21]\n"
96 "ld1 { v17.b }[14], [x20]\n"
97 "ld1 { v16.b }[14], [x19]\n"
98 "b 11f\n"
99 "4:" // odd_loads_1_12
100 "tbz %x[width], #0, 11f\n"
101 "ld1 { v19.b }[12], [x22]\n"
102 "ld1 { v18.b }[12], [x21]\n"
103 "ld1 { v17.b }[12], [x20]\n"
104 "ld1 { v16.b }[12], [x19]\n"
105 "b 11f\n"
106 "5:" // odd_loads_2_8
107 "tbz %x[width], #1, 6f\n"
108 "ld1 { v19.h }[4], [x22], #0x2\n"
109 "ld1 { v18.h }[4], [x21], #0x2\n"
110 "ld1 { v17.h }[4], [x20], #0x2\n"
111 "ld1 { v16.h }[4], [x19], #0x2\n"
112 "tbz %x[width], #0, 11f\n"
113 "ld1 { v19.b }[10], [x22]\n"
114 "ld1 { v18.b }[10], [x21]\n"
115 "ld1 { v17.b }[10], [x20]\n"
116 "ld1 { v16.b }[10], [x19]\n"
117 "b 11f\n"
118 "6:" // odd_loads_1_8
119 "tbz %x[width], #0, 11f\n"
120 "ld1 { v19.b }[8], [x22]\n"
121 "ld1 { v18.b }[8], [x21]\n"
122 "ld1 { v17.b }[8], [x20]\n"
123 "ld1 { v16.b }[8], [x19]\n"
124 "b 11f\n"
125 "7:" // odd_loads_4_0
126 "tbz %x[width], #2, 9f\n"
127 "ldr s19, [x22], #0x4\n"
128 "ldr s18, [x21], #0x4\n"
129 "ldr s17, [x20], #0x4\n"
130 "ldr s16, [x19], #0x4\n"
131 "tbz %x[width], #1, 8f\n"
132 "ld1 { v19.h }[2], [x22], #0x2\n"
133 "ld1 { v18.h }[2], [x21], #0x2\n"
134 "ld1 { v17.h }[2], [x20], #0x2\n"
135 "ld1 { v16.h }[2], [x19], #0x2\n"
136 "tbz %x[width], #0, 11f\n"
137 "ld1 { v19.b }[6], [x22]\n"
138 "ld1 { v18.b }[6], [x21]\n"
139 "ld1 { v17.b }[6], [x20]\n"
140 "ld1 { v16.b }[6], [x19]\n"
141 "b 11f\n"
142 "8:" // odd_loads_1_4
143 "tbz %x[width], #0, 11f\n"
144 "ld1 { v19.b }[4], [x22]\n"
145 "ld1 { v18.b }[4], [x21]\n"
146 "ld1 { v17.b }[4], [x20]\n"
147 "ld1 { v16.b }[4], [x19]\n"
148 "b 11f\n"
149 "9:" // odd_loads_2_0
150 "tbz %x[width], #1, 10f\n"
151 "ldr h19, [x22], #0x2\n"
152 "ldr h18, [x21], #0x2\n"
153 "ldr h17, [x20], #0x2\n"
154 "ldr h16, [x19], #0x2\n"
155 "tbz %x[width], #0, 11f\n"
156 "ld1 { v19.b }[2], [x22]\n"
157 "ld1 { v18.b }[2], [x21]\n"
158 "ld1 { v17.b }[2], [x20]\n"
159 "ld1 { v16.b }[2], [x19]\n"
160 "b 11f\n"
161 "10:" // odd_loads_1_0
162 "ldr b19, [x22, #0x0]\n"
163 "ldr b18, [x21, #0x0]\n"
164 "ldr b17, [x20, #0x0]\n"
165 "ldr b16, [x19, #0x0]\n"
166 "11:" // Odd load end
167 "str q19, [%x[out_ptr], #0x0]\n"
168 "str q18, [%x[out_ptr], #0x10]\n"
169 "str q17, [%x[out_ptr], #0x20]\n"
170 "str q16, [%x[out_ptr], #0x30]\n"
171 "add %x[out_ptr], %x[out_ptr], #0x40\n"
172 "12:" // Odds skip
173
174 : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width)
175 : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset)
176 : "cc", "memory", "v16", "v17", "v18", "v19", "x19", "x20", "x21", "x22"
177 );
178 }
179
180 template<>
interleave_block(uint8_t * & out_ptr,const uint8_t * const * in,size_t width,size_t height,size_t row_offset,bool)181 void interleave_block<4, 16, VLType::None, false>(
182 uint8_t * &out_ptr, const uint8_t * const * in, size_t width, size_t height,
183 size_t row_offset, bool
184 )
185 {
186 int8_t * &out_cast = reinterpret_cast<int8_t * &>(out_ptr);
187 const int8_t * const * in_cast = reinterpret_cast<const int8_t * const *>(in);
188
189 interleave_block<4, 16, VLType::None, false>(out_cast, in_cast, width, height, row_offset, false);
190 }
191
192
193 #endif // __aarch64__
194