1 /*
2 * Copyright (c) 2014 Travis Geiselbrecht
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #include <bits.h>
24 #include <ctype.h>
25 #include <debug.h>
26 #include <err.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <arch/arch_ops.h>
30 #include <arch/arm64.h>
31 #include <arch/mmu.h>
32 #include <arch/pan.h>
33 #include <arch/safecopy.h>
34 #include <kernel/vm.h>
35 #include <lib/trusty/trusty_app.h>
36 #include <inttypes.h>
37
38 #define SHUTDOWN_ON_FATAL 1
39
40 /**
41 * struct fault_handler_table_entry - Fault handler table entry.
42 * @pc: Address of the faulting instruction.
43 * @fault_handler: Address of the corresponding fault handler.
44 *
45 * Both addresses are position-relative, i.e., each field contains the offset
46 * from the field itself to its target.
47 */
48 struct fault_handler_table_entry {
49 int64_t pc;
50 int64_t fault_handler;
51 };
52
53 extern struct fault_handler_table_entry __fault_handler_table_start[];
54 extern struct fault_handler_table_entry __fault_handler_table_end[];
55
56 /**
57 * prel_to_abs_u64() - Convert a position-relative value to an absolute.
58 * @ptr: Pointer to a 64-bit position-relative value.
59 * @result: Pointer to the location for the result.
60 *
61 * Return: %true in case of success, %false for overflow.
62 */
prel_to_abs_u64(const int64_t * ptr,uint64_t * result)63 static inline bool prel_to_abs_u64(const int64_t* ptr, uint64_t* result) {
64 return !__builtin_add_overflow((uintptr_t)ptr, *ptr, result);
65 }
66
check_fault_handler_table(struct arm64_iframe_long * iframe)67 static bool check_fault_handler_table(struct arm64_iframe_long *iframe)
68 {
69 struct fault_handler_table_entry *fault_handler;
70 for (fault_handler = __fault_handler_table_start;
71 fault_handler < __fault_handler_table_end;
72 fault_handler++) {
73 uint64_t addr;
74 if (!prel_to_abs_u64(&fault_handler->pc, &addr)) {
75 /* Invalid entry, ignore it */
76 continue;
77 }
78 if (addr == iframe->elr) {
79 if (!prel_to_abs_u64(&fault_handler->fault_handler, &addr)) {
80 /*
81 * An entry with an invalid handler address. We don't expect
82 * another entry with the same pc, so we break out of
83 * the loop early.
84 */
85 return false;
86 }
87
88 iframe->elr = addr;
89 return true;
90 }
91 }
92 return false;
93 }
94
95 #if TEST_BUILD
wrap_add(uint64_t addr,int offset)96 static uint64_t wrap_add(uint64_t addr, int offset) {
97 uint64_t result;
98 __builtin_add_overflow(addr, offset, &result);
99 return result;
100 }
101
getmeminfo(uint64_t addr,paddr_t * paddr,uint * flags,char * name,size_t name_size)102 static bool getmeminfo(uint64_t addr, paddr_t *paddr, uint *flags, char *name,
103 size_t name_size) {
104 status_t ret = NO_ERROR;
105 vmm_aspace_t *aspace = vaddr_to_aspace((void*)addr);
106 if (aspace) {
107 ret = arch_mmu_query(&aspace->arch_aspace, addr, paddr, flags);
108 }
109
110 if (aspace && ret == NO_ERROR) {
111 if (name) {
112 vmm_get_address_description(addr, name, name_size);
113 }
114 return true;
115 }
116 return false;
117 }
118
printmemattrs(const char * prefix,paddr_t pstart,vaddr_t vstart,size_t len,uint flags,const char * name)119 static void printmemattrs(
120 const char *prefix, paddr_t pstart, vaddr_t vstart, size_t len,
121 uint flags, const char* name) {
122 if (!len) {
123 return;
124 }
125 printf("%s0x%lx/0x%zx, flags: 0x%02x, region: %s [ read%s%s%s",
126 prefix, pstart, len, flags, name,
127 !(flags & ARCH_MMU_FLAG_PERM_RO) ? " write" : "",
128 !(flags & ARCH_MMU_FLAG_PERM_NO_EXECUTE) ? " execute" : "",
129 (flags & ARCH_MMU_FLAG_PERM_USER) ? " user" : "");
130 if (flags & ARCH_MMU_FLAG_TAGGED) {
131 printf(" tagged(");
132 /*
133 * There is one tag for every 16-byte aligned 16 bytes, so depending
134 * on len and the alignment of vstart, there may be one extra tag.
135 */
136 int numtags = 1 + (round_down(vstart + (len - 1), MTE_GRANULE_SIZE) -
137 round_down(vstart, MTE_GRANULE_SIZE)) / MTE_GRANULE_SIZE;
138 for (int i = 0; i < numtags; i++) {
139 if (i) {
140 printf("/");
141 }
142 int tag = tag_for_address(vstart + i * MTE_GRANULE_SIZE);
143 if (tag < 0) {
144 printf("?");
145 } else {
146 printf("%d", tag);
147 }
148 }
149 printf(")");
150 }
151
152 printf("%s%s%s ]):\n",
153 (flags & ARCH_MMU_FLAG_NS) ? " nonsecure" : "",
154 (flags & ARCH_MMU_FLAG_UNCACHED_DEVICE) ? " device" : "",
155 (flags & ARCH_MMU_FLAG_UNCACHED) ? " uncached" : "");
156 }
157
dump_memory_around_register(const char * name,uint64_t regaddr)158 static void dump_memory_around_register(const char *name, uint64_t regaddr) {
159 uint64_t addr = wrap_add(regaddr, -16);
160 uint64_t secondpageaddr;
161
162 uint8_t data[48];
163
164 int page_size = PAGE_SIZE;
165 if (is_user_address(addr)) {
166 page_size = USER_PAGE_SIZE;
167 }
168 uint64_t offsetinpage = addr & (page_size - 1);
169 uint64_t bytesonfirstpage = page_size - offsetinpage;
170 if (bytesonfirstpage > sizeof(data)) {
171 bytesonfirstpage = sizeof(data);
172 }
173
174 paddr_t paddr1, paddr2;
175 uint flags1, flags2;
176 char name1[VMM_MAX_ADDRESS_DESCRIPTION_SIZE];
177 char name2[VMM_MAX_ADDRESS_DESCRIPTION_SIZE];
178 bool info1valid =false;
179 bool info2valid = false;
180 bool read1ok = false;
181 bool read2ok = false;
182
183 info1valid = getmeminfo(addr, &paddr1, &flags1, name1, sizeof(name1));
184
185 if (bytesonfirstpage < sizeof(data)) {
186 /* this block spans a page boundary */
187 secondpageaddr = wrap_add(addr, bytesonfirstpage);
188 info2valid = getmeminfo(secondpageaddr, &paddr2, &flags2, name2,
189 sizeof(name2));
190 }
191
192 if (!info1valid && !info2valid) {
193 return;
194 }
195
196 if (info1valid &&
197 ((flags1 & ARCH_MMU_FLAG_CACHE_MASK) == ARCH_MMU_FLAG_CACHED ||
198 (flags1 & ARCH_MMU_FLAG_CACHE_MASK) == ARCH_MMU_FLAG_UNCACHED)) {
199 /* this should only fail if the page was remapped after we queried it */
200 status_t ret = copy_from_anywhere(data, addr, bytesonfirstpage);
201 read1ok = (ret == NO_ERROR);
202 }
203
204 if (info2valid &&
205 ((flags2 & ARCH_MMU_FLAG_CACHE_MASK) == ARCH_MMU_FLAG_CACHED ||
206 (flags2 & ARCH_MMU_FLAG_CACHE_MASK) == ARCH_MMU_FLAG_UNCACHED)) {
207 status_t ret = copy_from_anywhere(data + bytesonfirstpage,
208 secondpageaddr, sizeof(data) - bytesonfirstpage);
209 read2ok = (ret == NO_ERROR);
210 }
211
212 printf(" \nmemory around %3s (", name);
213 if (info1valid) {
214 printmemattrs("phys: ", paddr1, addr, bytesonfirstpage, flags1, name1);
215 } else {
216 printf("phys: <unmapped>/0x%" PRIx64 "):\n", bytesonfirstpage);
217 }
218 if (bytesonfirstpage < sizeof(data)) {
219 if (info2valid) {
220 printmemattrs(" and (phys: ",
221 paddr2,
222 secondpageaddr,
223 sizeof(data) - bytesonfirstpage,
224 flags2,
225 name2);
226 } else {
227 printf(" and (phys: <unmapped>/0x%" PRIx64 "):\n",
228 sizeof(data) - bytesonfirstpage);
229 }
230 }
231
232 const int linelen = 16;
233 int tag_granule_offset = (addr & (MTE_GRANULE_SIZE - 1));
234 for (size_t offset = 0; offset < sizeof(data); offset += linelen) {
235 printf("0x%016" PRIx64 ": ", wrap_add(addr, offset));
236
237 for (int i = 0; i < linelen; i++) {
238 if (i == 8) {
239 printf(" ");
240 }
241 if ((offset + i < bytesonfirstpage && read1ok) ||
242 (offset + i >= bytesonfirstpage && read2ok)) {
243 printf("%02hhx", data[offset + i]);
244 if (i != (linelen - 1) &&
245 ((offset + i + tag_granule_offset) %
246 MTE_GRANULE_SIZE) == (MTE_GRANULE_SIZE - 1) &&
247 ((offset + i < bytesonfirstpage) ? flags1 : flags2) &
248 ARCH_MMU_FLAG_TAGGED) {
249 printf("/");
250 } else {
251 printf(" ");
252 }
253 } else {
254 printf("-- ");
255 }
256 }
257
258 printf("|");
259
260 for (int i = 0; i < linelen; i++) {
261 unsigned char c = data[offset + i];
262 printf("%c", ((offset + i < bytesonfirstpage && read1ok) ||
263 (offset + i >= bytesonfirstpage && read2ok)) &&
264 isprint(c) ? c : '.');
265 }
266
267 printf("\n");
268 }
269 }
270
dump_memory_around_registers(const struct arm64_iframe_long * iframe)271 static void dump_memory_around_registers(
272 const struct arm64_iframe_long *iframe) {
273 char regname[4];
274 for (int i = 0; i < 28; i++) {
275 snprintf(regname, sizeof(regname), "x%d", i);
276 dump_memory_around_register(regname, iframe->r[i]);
277 }
278 dump_memory_around_register("fp", iframe->fp);
279 dump_memory_around_register("lr", arch_extract_return_addr(iframe->lr));
280 dump_memory_around_register("sp", iframe->sp);
281 dump_memory_around_register("elr", iframe->elr);
282 }
283 #endif
284
dump_iframe(const struct arm64_iframe_long * iframe)285 static void dump_iframe(const struct arm64_iframe_long *iframe)
286 {
287 struct thread *thread = get_current_thread();
288 printf("thread: %p (%s)\n", thread, thread->name);
289 printf("stack %p-%p\n", thread->stack, thread->stack + thread->stack_size);
290 printf("iframe %p:\n", iframe);
291 #if TEST_BUILD
292 printf("x0 0x%16" PRIx64 " x1 0x%16" PRIx64 " x2 0x%16" PRIx64 " x3 0x%16" PRIx64 "\n", iframe->r[0], iframe->r[1], iframe->r[2], iframe->r[3]);
293 printf("x4 0x%16" PRIx64 " x5 0x%16" PRIx64 " x6 0x%16" PRIx64 " x7 0x%16" PRIx64 "\n", iframe->r[4], iframe->r[5], iframe->r[6], iframe->r[7]);
294 printf("x8 0x%16" PRIx64 " x9 0x%16" PRIx64 " x10 0x%16" PRIx64 " x11 0x%16" PRIx64 "\n", iframe->r[8], iframe->r[9], iframe->r[10], iframe->r[11]);
295 printf("x12 0x%16" PRIx64 " x13 0x%16" PRIx64 " x14 0x%16" PRIx64 " x15 0x%16" PRIx64 "\n", iframe->r[12], iframe->r[13], iframe->r[14], iframe->r[15]);
296 printf("x16 0x%16" PRIx64 " x17 0x%16" PRIx64 " x18 0x%16" PRIx64 " x19 0x%16" PRIx64 "\n", iframe->r[16], iframe->r[17], iframe->r[18], iframe->r[19]);
297 printf("x20 0x%16" PRIx64 " x21 0x%16" PRIx64 " x22 0x%16" PRIx64 " x23 0x%16" PRIx64 "\n", iframe->r[20], iframe->r[21], iframe->r[22], iframe->r[23]);
298 printf("x24 0x%16" PRIx64 " x25 0x%16" PRIx64 " x26 0x%16" PRIx64 " x27 0x%16" PRIx64 "\n", iframe->r[24], iframe->r[25], iframe->r[26], iframe->r[27]);
299 printf("x28 0x%16" PRIx64 " fp 0x%16" PRIx64 " lr 0x%16" PRIx64 " sp 0x%16" PRIx64 "\n", iframe->r[28], iframe->fp, iframe->lr, iframe->sp);
300
301 /* Check if lr contains a PAC and also display the original */
302 uintptr_t lr_xpac = arch_extract_return_addr(iframe->lr);
303 if (lr_xpac != iframe->lr) {
304 printf("lr 0x%16" PRIx64 " (pac removed)\n", lr_xpac);
305 }
306
307 printf("elr 0x%16" PRIx64 "\n", iframe->elr);
308 printf("spsr 0x%16" PRIx64 "\n", iframe->spsr);
309 #endif
310 }
311
arm64_syscall(struct arm64_iframe_long * iframe,bool is_64bit)312 __WEAK void arm64_syscall(struct arm64_iframe_long *iframe, bool is_64bit)
313 {
314 panic("unhandled syscall vector\n");
315 }
316
print_fault_code(uint32_t fsc,uint64_t far)317 static void print_fault_code(uint32_t fsc, uint64_t far) {
318 printf("fault code 0x%x: ", fsc);
319 switch (fsc) {
320 case 0b000000:
321 case 0b000001:
322 case 0b000010:
323 case 0b000011:
324 printf("Address size fault, level %d", fsc & 0x3);
325 break;
326 case 0b000100:
327 case 0b000101:
328 case 0b000110:
329 case 0b000111:
330 printf("Translation fault, level %d", fsc & 0x3);
331 break;
332 case 0b001001:
333 case 0b001010:
334 case 0b001011:
335 printf("Access flag fault, level %d", fsc & 0x3);
336 break;
337 case 0b001101:
338 case 0b001110:
339 case 0b001111:
340 printf("Permission fault, level %d", fsc & 0x3);
341 break;
342
343 case 0b010000:
344 printf("External abort");
345 break;
346
347 case 0b010001: {
348 printf("Tag check fault");
349 #if TEST_BUILD
350 {
351 uint64_t used_tag = (far >> 56) & 0xf;
352 int real_tag = tag_for_address(far);
353 printf(": %" PRIu64 " should be ", used_tag);
354 if (real_tag < 0) {
355 printf(" ? (faulted while trying to read actual tag)");
356 } else {
357 printf("%d", real_tag);
358 }
359 }
360 #endif
361 break;
362 }
363
364 case 0b010100:
365 case 0b010101:
366 case 0b010110:
367 case 0b010111:
368 printf("External abort on translation table, level %d", fsc & 0x3);
369 break;
370
371 case 0b011000:
372 printf("Parity or ECC error");
373 break;
374
375 case 0b011100:
376 case 0b011101:
377 case 0b011110:
378 case 0b011111:
379 printf("Parity or ECC error on translation table, level %d", fsc & 0x3);
380 break;
381
382 case 0b100001:
383 printf("Alignment fault");
384 break;
385
386 case 0b110000:
387 printf("TLB conflict abort");
388 break;
389
390 case 0b110001:
391 printf("Unsupported atomic hardware update fault");
392 break;
393
394 case 0b110100:
395 printf("Lockdown fault");
396 break;
397
398 case 0b110101:
399 printf("Unsupported exclusive or atomic access");
400 break;
401
402 default:
403 printf("Unknown fault");
404 break;
405 }
406 printf("\n");
407 }
408
enable_tag_checks(void)409 static void enable_tag_checks(void)
410 {
411 if (arm64_tagging_supported()) {
412 /* Clearing Tag Check Override enables tag checking */
413 __asm__ volatile(".arch_extension memtag\n"
414 "msr tco, #0");
415 }
416 }
417
disable_tag_checks(void)418 static void disable_tag_checks(void)
419 {
420 if (arm64_tagging_supported()) {
421 /* Setting Tag Check Override disables tag checking */
422 __asm__ volatile(".arch_extension memtag\n"
423 "msr tco, #1");
424 }
425 }
arm64_sync_exception(struct arm64_iframe_long * iframe,bool from_lower)426 void arm64_sync_exception(struct arm64_iframe_long *iframe, bool from_lower)
427 {
428 uint32_t esr = ARM64_READ_SYSREG(esr_el1);
429 uint32_t ec = BITS_SHIFT(esr, 31, 26);
430 uint32_t il = BIT_SHIFT(esr, 25);
431 uint32_t iss = BITS(esr, 24, 0);
432 uintptr_t display_pc = iframe->elr;
433 uint64_t far;
434 __UNUSED bool print_mem_around_fault = false;
435
436 /*
437 * Tag checks are automatically disabled on taking an exception, so
438 * turn them back on
439 */
440 enable_tag_checks();
441
442 if (from_lower) {
443 /*
444 * load_bias may intentionally overflow to represent a shift
445 * down of the application base address
446 */
447 __builtin_sub_overflow(display_pc,
448 current_trusty_app()->load_bias,
449 &display_pc);
450 }
451
452 switch (ec) {
453 case 0b000111: /* floating point */
454 arm64_fpu_exception(iframe);
455 return;
456 case 0b010001: /* syscall from arm32 */
457 case 0b010101: /* syscall from arm64 */
458 #ifdef WITH_LIB_SYSCALL
459 arch_enable_fiqs();
460 arm64_syscall(iframe, (ec == 0x15) ? true : false);
461 arch_disable_fiqs();
462 return;
463 #else
464 arm64_syscall(iframe, (ec == 0x15) ? true : false);
465 return;
466 #endif
467 case 0b100000: /* instruction abort from lower level */
468 case 0b100001: /* instruction abort from same level */
469 if (check_fault_handler_table(iframe)) {
470 return;
471 }
472 printf("instruction abort: PC at 0x%" PRIx64 "(0x%lx)\n", iframe->elr,
473 display_pc);
474 print_fault_code(BITS(iss, 5, 0), 0);
475 break;
476 case 0b100100: /* data abort from lower level */
477 case 0b100101: { /* data abort from same level */
478 if (check_fault_handler_table(iframe)) {
479 return;
480 }
481
482 /* read the FAR register */
483 far = ARM64_READ_SYSREG(far_el1);
484
485 /* decode the iss */
486 uint32_t dfsc = BITS(iss, 5, 0);
487 printf("data fault ");
488 if (BIT(iss, 6)) {
489 printf("writing to ");
490 } else {
491 printf("reading from ");
492 }
493 if (dfsc == 0b010000 && BIT(iss, 10)) {
494 printf("unknown address (FAR 0x%" PRIx64 " not valid)", far);
495 } else {
496 printf("0x%" PRIx64, far);
497 print_mem_around_fault = true;
498 vmm_aspace_t *aspace = vaddr_to_aspace((void*)far);
499 if (aspace) {
500 char region[VMM_MAX_ADDRESS_DESCRIPTION_SIZE];
501 vmm_get_address_description(far, region, sizeof(region));
502 printf(" (%s)", region);
503 }
504 }
505 printf(", PC at 0x%" PRIx64 "(0x%lx)\n", iframe->elr, display_pc);
506 if (BIT(iss, 24)) { /* ISV bit */
507 printf("Access size: %d bits, sign extension: %s, register: %s%lu, %s acquire release semantics\n",
508 8 << BITS_SHIFT(iss,23,22),
509 BIT_SHIFT(iss,21) ? "yes" : "no",
510 BIT_SHIFT(iss,15) ? "X" : "W",
511 BITS_SHIFT(iss,20,16),
512 BIT_SHIFT(iss,14) ? "" : "no");
513 }
514 print_fault_code(dfsc, far);
515 break;
516 }
517 case 0b001101: { /* branch target exception */
518 #if TEST_BUILD
519 if (check_fault_handler_table(iframe)) {
520 return;
521 }
522 #endif
523 printf("branch target exception: BTYPE=0x%lx, "
524 "PC at 0x%" PRIx64 "(0x%lx)\n", BITS(iss, 1, 0), iframe->elr, display_pc);
525 break;
526 }
527 case 0b011100: { /* FPAC exception */
528 #if TEST_BUILD
529 if (check_fault_handler_table(iframe)) {
530 return;
531 }
532 #endif
533 printf("PAC auth failure target exception: instr_type=%c%c, "
534 "PC at 0x%" PRIx64 "(0x%lx)\n",
535 BIT(iss, 1) ? 'D' : 'I', /* data / instruction */
536 BIT(iss, 0) ? 'B' : 'A', /* key */
537 iframe->elr, display_pc);
538 break;
539 }
540 case 0b111100: {
541 printf("BRK #0x%04lx instruction: PC at 0x%" PRIx64 "(0x%lx)\n",
542 BITS_SHIFT(iss, 15, 0), iframe->elr, display_pc);
543 break;
544 }
545 default:
546 printf("unhandled synchronous exception: PC at 0x%" PRIx64 "(0x%lx)\n",
547 iframe->elr, display_pc);
548 }
549
550 /* unhandled exception, die here */
551 if (from_lower) {
552 printf("app: %s\n", current_trusty_app()->props.app_name);
553 printf("load bias: 0x%lx\n", current_trusty_app()->load_bias);
554 }
555 printf("ESR 0x%x: ec 0x%x, il 0x%x, iss 0x%x\n", esr, ec, il, iss);
556 dump_iframe(iframe);
557 #if TEST_BUILD
558 disable_tag_checks();
559 dump_memory_around_registers(iframe);
560 if (print_mem_around_fault) {
561 dump_memory_around_register("fault address", far);
562 }
563 enable_tag_checks();
564 #endif
565
566 if (from_lower) {
567 arch_enable_fiqs();
568 arch_enable_ints();
569 /* TODO(snehalreddy): Remove ASLR */
570 trusty_app_crash(esr, far, display_pc);
571 }
572 panic("die\n");
573 }
574
arm64_invalid_exception(struct arm64_iframe_long * iframe,unsigned int which)575 void arm64_invalid_exception(struct arm64_iframe_long *iframe, unsigned int which)
576 {
577 printf("invalid exception, which 0x%x\n", which);
578 dump_iframe(iframe);
579 #if TEST_BUILD
580 dump_memory_around_registers(iframe);
581 #endif
582
583 panic("die\n");
584 }
585