1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <fcntl.h>
30 #include <private/bionic_ifuncs.h>
31 #include <sys/syscall.h>
32
33 extern "C" {
34
35 enum CpuVariant {
36 kUnknown = 0,
37 kGeneric,
38 kCortexA7,
39 kCortexA9,
40 kCortexA53,
41 kCortexA55,
42 kKrait,
43 kKryo,
44 };
45
46 static constexpr int MAX_CPU_NAME_LEN = 12;
47 struct CpuVariantNames {
48 alignas(alignof(int)) char name[MAX_CPU_NAME_LEN];
49 CpuVariant variant;
50 };
51
52 static constexpr CpuVariantNames cpu_variant_names[] = {
53 {"cortex-a76", kCortexA55},
54 {"kryo385", kCortexA55},
55 {"cortex-a75", kCortexA55},
56 {"kryo", kKryo},
57 {"cortex-a73", kCortexA55},
58 {"cortex-a55", kCortexA55},
59 {"cortex-a53", kCortexA53},
60 {"krait", kKrait},
61 {"cortex-a9", kCortexA9},
62 {"cortex-a7", kCortexA7},
63 // kUnknown indicates the end of this array.
64 {"", kUnknown},
65 };
66
ifunc_open(const char * pathname)67 static long ifunc_open(const char* pathname) {
68 register long r0 __asm__("r0") = AT_FDCWD;
69 register long r1 __asm__("r1") = reinterpret_cast<long>(pathname);
70 register long r2 __asm__("r2") = O_RDONLY;
71 register long r3 __asm__("r3") = 0;
72 register long r7 __asm__("r7") = __NR_openat;
73 __asm__ volatile("swi #0"
74 : "=r"(r0)
75 : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r7));
76 return r0;
77 }
78
ifunc_read(int fd,void * buf,size_t count)79 static ssize_t ifunc_read(int fd, void* buf, size_t count) {
80 register long r0 __asm__("r0") = fd;
81 register long r1 __asm__("r1") = reinterpret_cast<long>(buf);
82 register long r2 __asm__("r2") = count;
83 register long r7 __asm__("r7") = __NR_read;
84 __asm__ volatile("swi #0"
85 : "=r"(r0)
86 : "r"(r0), "r"(r1), "r"(r2), "r"(r7)
87 : "memory");
88 return r0;
89 }
90
ifunc_close(int fd)91 static int ifunc_close(int fd) {
92 register long r0 __asm__("r0") = fd;
93 register long r7 __asm__("r7") = __NR_close;
94 __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r7));
95 return r0;
96 }
97
is_same_name(const char * a,const char * b)98 static bool is_same_name(const char* a, const char* b) {
99 static_assert(MAX_CPU_NAME_LEN % sizeof(int) == 0, "");
100 const int* ia = reinterpret_cast<const int*>(a);
101 const int* ib = reinterpret_cast<const int*>(b);
102 for (size_t i = 0; i < MAX_CPU_NAME_LEN / sizeof(int); ++i) {
103 if (ia[i] != ib[i]) {
104 return false;
105 }
106 }
107 return true;
108 }
109
init_cpu_variant()110 static CpuVariant init_cpu_variant() {
111 int fd = ifunc_open("/dev/cpu_variant:arm");
112 if (fd < 0) return kGeneric;
113
114 alignas(alignof(int)) char name[MAX_CPU_NAME_LEN] = {};
115
116 int bytes_read, total_read = 0;
117 while (total_read < MAX_CPU_NAME_LEN - 1 &&
118 (bytes_read = ifunc_read(fd, name + total_read,
119 MAX_CPU_NAME_LEN - 1 - total_read)) > 0) {
120 total_read += bytes_read;
121 }
122 ifunc_close(fd);
123
124 if (bytes_read != 0) {
125 // The file is too big. We haven't reach the end. Or maybe there is an
126 // error when reading.
127 return kGeneric;
128 }
129 name[total_read] = 0;
130
131 const CpuVariantNames* cpu_variant = cpu_variant_names;
132 while (cpu_variant->variant != kUnknown) {
133 if (is_same_name(cpu_variant->name, name)) {
134 return cpu_variant->variant;
135 }
136 cpu_variant++;
137 }
138 return kGeneric;
139 }
140
get_cpu_variant()141 static CpuVariant get_cpu_variant() {
142 static CpuVariant cpu_variant = kUnknown;
143 if (cpu_variant == kUnknown) {
144 cpu_variant = init_cpu_variant();
145 }
146 return cpu_variant;
147 }
148
DEFINE_IFUNC_FOR(memmove)149 DEFINE_IFUNC_FOR(memmove) {
150 RETURN_FUNC(memmove_func_t, memmove_a15);
151 }
152 MEMMOVE_SHIM()
153
DEFINE_IFUNC_FOR(memcpy)154 DEFINE_IFUNC_FOR(memcpy) {
155 return memmove_resolver(hwcap);
156 }
157 MEMCPY_SHIM()
158
159 // On arm32, __memcpy() is not publicly exposed, but gets called by memmove()
160 // in cases where the copy is known to be overlap-safe.
161 typedef void* __memcpy_func_t(void*, const void*, size_t);
DEFINE_IFUNC_FOR(__memcpy)162 DEFINE_IFUNC_FOR(__memcpy) {
163 switch (get_cpu_variant()) {
164 case kCortexA7:
165 RETURN_FUNC(__memcpy_func_t, __memcpy_a7);
166 case kCortexA9:
167 RETURN_FUNC(__memcpy_func_t, __memcpy_a9);
168 case kKrait:
169 RETURN_FUNC(__memcpy_func_t, __memcpy_krait);
170 case kCortexA53:
171 RETURN_FUNC(__memcpy_func_t, __memcpy_a53);
172 case kCortexA55:
173 RETURN_FUNC(__memcpy_func_t, __memcpy_a55);
174 case kKryo:
175 RETURN_FUNC(__memcpy_func_t, __memcpy_kryo);
176 default:
177 RETURN_FUNC(__memcpy_func_t, __memcpy_a15);
178 }
179 }
180 DEFINE_STATIC_SHIM(void* __memcpy(void* dst, const void* src, size_t n) {
181 FORWARD(__memcpy)(dst, src, n);
182 })
183
DEFINE_IFUNC_FOR(__memset_chk)184 DEFINE_IFUNC_FOR(__memset_chk) {
185 switch (get_cpu_variant()) {
186 case kCortexA7:
187 case kCortexA53:
188 case kCortexA55:
189 case kKryo:
190 RETURN_FUNC(__memset_chk_func_t, __memset_chk_a7);
191 case kCortexA9:
192 RETURN_FUNC(__memset_chk_func_t, __memset_chk_a9);
193 case kKrait:
194 RETURN_FUNC(__memset_chk_func_t, __memset_chk_krait);
195 default:
196 RETURN_FUNC(__memset_chk_func_t, __memset_chk_a15);
197 }
198 }
199 __MEMSET_CHK_SHIM()
200
DEFINE_IFUNC_FOR(memset)201 DEFINE_IFUNC_FOR(memset) {
202 switch (get_cpu_variant()) {
203 case kCortexA7:
204 case kCortexA53:
205 case kCortexA55:
206 case kKryo:
207 RETURN_FUNC(memset_func_t, memset_a7);
208 case kCortexA9:
209 RETURN_FUNC(memset_func_t, memset_a9);
210 case kKrait:
211 RETURN_FUNC(memset_func_t, memset_krait);
212 default:
213 RETURN_FUNC(memset_func_t, memset_a15);
214 }
215 }
216 MEMSET_SHIM()
217
DEFINE_IFUNC_FOR(strcpy)218 DEFINE_IFUNC_FOR(strcpy) {
219 switch (get_cpu_variant()) {
220 case kCortexA9:
221 RETURN_FUNC(strcpy_func_t, strcpy_a9);
222 default:
223 RETURN_FUNC(strcpy_func_t, strcpy_a15);
224 }
225 }
226 STRCPY_SHIM()
227
DEFINE_IFUNC_FOR(__strcpy_chk)228 DEFINE_IFUNC_FOR(__strcpy_chk) {
229 switch (get_cpu_variant()) {
230 case kCortexA7:
231 RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a7);
232 case kCortexA9:
233 RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a9);
234 case kKrait:
235 case kKryo:
236 RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_krait);
237 case kCortexA53:
238 RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a53);
239 case kCortexA55:
240 RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a55);
241 default:
242 RETURN_FUNC(__strcpy_chk_func_t, __strcpy_chk_a15);
243 }
244 }
245 __STRCPY_CHK_SHIM()
246
DEFINE_IFUNC_FOR(stpcpy)247 DEFINE_IFUNC_FOR(stpcpy) {
248 switch (get_cpu_variant()) {
249 case kCortexA9:
250 RETURN_FUNC(stpcpy_func_t, stpcpy_a9);
251 default:
252 RETURN_FUNC(stpcpy_func_t, stpcpy_a15);
253 }
254 }
255 STPCPY_SHIM()
256
DEFINE_IFUNC_FOR(strcat)257 DEFINE_IFUNC_FOR(strcat) {
258 switch (get_cpu_variant()) {
259 case kCortexA9:
260 RETURN_FUNC(strcat_func_t, strcat_a9);
261 default:
262 RETURN_FUNC(strcat_func_t, strcat_a15);
263 }
264 }
265 STRCAT_SHIM()
266
DEFINE_IFUNC_FOR(__strcat_chk)267 DEFINE_IFUNC_FOR(__strcat_chk) {
268 switch (get_cpu_variant()) {
269 case kCortexA7:
270 RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a7);
271 case kCortexA9:
272 RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a9);
273 case kKrait:
274 case kKryo:
275 RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_krait);
276 case kCortexA53:
277 RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a53);
278 case kCortexA55:
279 RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a55);
280 default:
281 RETURN_FUNC(__strcat_chk_func_t, __strcat_chk_a15);
282 }
283 }
284 __STRCAT_CHK_SHIM()
285
DEFINE_IFUNC_FOR(strcmp)286 DEFINE_IFUNC_FOR(strcmp) {
287 RETURN_FUNC(strcmp_func_t, strcmp_a15);
288 }
289 STRCMP_SHIM()
290
DEFINE_IFUNC_FOR(strlen)291 DEFINE_IFUNC_FOR(strlen) {
292 switch (get_cpu_variant()) {
293 case kCortexA9:
294 RETURN_FUNC(strlen_func_t, strlen_a9);
295 default:
296 RETURN_FUNC(strlen_func_t, strlen_a15);
297 }
298 }
299 STRLEN_SHIM()
300
301 } // extern "C"
302