1 /*
2 * Copyright (c) 2014 Google Inc. All rights reserved
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <arch/arm64/mmu.h>
25 #include <arch/ops.h>
26 #include <assert.h>
27 #include <bits.h>
28 #include <debug.h>
29 #include <err.h>
30 #include <kernel/thread.h>
31 #include <kernel/vm.h>
32 #include <lib/heap.h>
33 #include <stdbool.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <sys/types.h>
37 #include <trace.h>
38 #include <inttypes.h>
39
40 #define LOCAL_TRACE 0
41 #define TRACE_CONTEXT_SWITCH 0
42
43 #define ARM64_ASID_BITS (8) /* TODO: Use 16 bit ASIDs when hardware supports it */
44
45 STATIC_ASSERT(((long)KERNEL_BASE >> MMU_KERNEL_SIZE_SHIFT) == -1);
46 STATIC_ASSERT(((long)KERNEL_ASPACE_BASE >> MMU_KERNEL_SIZE_SHIFT) == -1);
47 STATIC_ASSERT(MMU_KERNEL_SIZE_SHIFT <= 48);
48 STATIC_ASSERT(MMU_KERNEL_SIZE_SHIFT >= 25);
49 STATIC_ASSERT(USER_ASPACE_BASE + USER_ASPACE_SIZE <= 1UL << MMU_USER_SIZE_SHIFT);
50
51 /* the main translation table */
52 extern pte_t arm64_kernel_translation_table[];
53
54 /* This is explicitly a check for overflows, so don't sanitize it */
55 __attribute__((no_sanitize("unsigned-integer-overflow")))
wrap_check(vaddr_t vaddr,size_t size)56 static inline bool wrap_check(vaddr_t vaddr, size_t size) {
57 return vaddr + size - 1 > vaddr;
58 }
59
arch_mmu_asid(arch_aspace_t * aspace)60 static uint64_t arch_mmu_asid(arch_aspace_t *aspace)
61 {
62 return aspace->asid & BIT_MASK(ARM64_ASID_BITS);
63 }
64
adjusted_vaddr(arch_aspace_t * aspace,vaddr_t vaddr)65 static inline vaddr_t adjusted_vaddr(arch_aspace_t *aspace, vaddr_t vaddr)
66 {
67 return arch_adjusted_vaddr(vaddr, aspace->flags & ARCH_ASPACE_FLAG_KERNEL);
68 }
69
is_valid_vaddr(arch_aspace_t * aspace,vaddr_t vaddr)70 static inline bool is_valid_vaddr(arch_aspace_t *aspace, vaddr_t vaddr)
71 {
72 vaddr = adjusted_vaddr(aspace, vaddr);
73 return (aspace->size && vaddr >= aspace->base && vaddr <= aspace->base + (aspace->size - 1));
74 }
75
76 /* convert user level mmu flags to flags that go in L1 descriptors */
mmu_flags_to_pte_attr(const uint aspace_flags,const uint flags,pte_t * out_attr)77 static bool mmu_flags_to_pte_attr(const uint aspace_flags, const uint flags, pte_t* out_attr)
78 {
79 pte_t attr = MMU_PTE_ATTR_AF;
80
81 DEBUG_ASSERT((aspace_flags & ~ARCH_ASPACE_FLAG_ALL) == 0);
82
83 /* Enforce that executable mappings are not also writable */
84 if ((flags & (ARCH_MMU_FLAG_PERM_RO | ARCH_MMU_FLAG_PERM_NO_EXECUTE)) == 0) {
85 return false;
86 }
87
88 if (flags & ARCH_MMU_FLAG_TAGGED) {
89 if ((flags & ARCH_MMU_FLAG_CACHE_MASK) & ~ARCH_MMU_FLAG_CACHED) {
90 /* only normal memory can be tagged */
91 return false;
92 }
93 }
94
95 switch (flags & ARCH_MMU_FLAG_CACHE_MASK) {
96 case ARCH_MMU_FLAG_CACHED:
97 if (flags & ARCH_MMU_FLAG_TAGGED) {
98 attr |= MMU_PTE_ATTR_NORMAL_MEMORY_TAGGED;
99 } else {
100 attr |= MMU_PTE_ATTR_NORMAL_MEMORY;
101 }
102 attr |= MMU_PTE_ATTR_SH_INNER_SHAREABLE;
103 break;
104 case ARCH_MMU_FLAG_UNCACHED:
105 attr |= MMU_PTE_ATTR_STRONGLY_ORDERED;
106 break;
107 case ARCH_MMU_FLAG_UNCACHED_DEVICE:
108 attr |= MMU_PTE_ATTR_DEVICE;
109 break;
110 default:
111 /* invalid user-supplied flag */
112 DEBUG_ASSERT(0);
113 return false;
114 }
115
116 switch (flags & (ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_RO)) {
117 case 0:
118 attr |= MMU_PTE_ATTR_AP_P_RW_U_NA;
119 break;
120 case ARCH_MMU_FLAG_PERM_RO:
121 attr |= MMU_PTE_ATTR_AP_P_RO_U_NA;
122 break;
123 case ARCH_MMU_FLAG_PERM_USER:
124 attr |= MMU_PTE_ATTR_AP_P_RW_U_RW;
125 break;
126 case ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_RO:
127 attr |= MMU_PTE_ATTR_AP_P_RO_U_RO;
128 break;
129 }
130
131 if (flags & ARCH_MMU_FLAG_PERM_NO_EXECUTE) {
132 attr |= MMU_PTE_ATTR_UXN | MMU_PTE_ATTR_PXN;
133 } else if (flags & ARCH_MMU_FLAG_PERM_USER) {
134 /* User executable page, marked privileged execute never. */
135 attr |= MMU_PTE_ATTR_PXN;
136
137 if (aspace_flags & ARCH_ASPACE_FLAG_BTI) {
138 attr |= MMU_PTE_ATTR_GP;
139 }
140 } else {
141 /* Privileged executable page, marked user execute never. */
142 attr |= MMU_PTE_ATTR_UXN;
143
144 if (aspace_flags & ARCH_ASPACE_FLAG_BTI) {
145 attr |= MMU_PTE_ATTR_GP;
146 }
147 }
148
149 if (flags & ARCH_MMU_FLAG_NS) {
150 attr |= MMU_PTE_ATTR_NON_SECURE;
151 }
152
153 /* GP bit is undefined/MBZ if BTI is not supported */
154 if ((attr & MMU_PTE_ATTR_GP) && !arch_bti_supported()) {
155 return false;
156 }
157
158 *out_attr = attr;
159 return true;
160 }
161
162 #ifndef EARLY_MMU
arch_mmu_query(arch_aspace_t * aspace,vaddr_t vaddr,paddr_t * paddr,uint * flags)163 status_t arch_mmu_query(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t *paddr, uint *flags)
164 {
165 uint index;
166 uint index_shift;
167 uint page_size_shift;
168 pte_t pte;
169 pte_t pte_addr;
170 uint descriptor_type;
171 pte_t *page_table;
172 vaddr_t vaddr_rem;
173
174 LTRACEF("aspace %p, vaddr 0x%lx\n", aspace, vaddr);
175
176 DEBUG_ASSERT(aspace);
177 DEBUG_ASSERT(aspace->tt_virt);
178
179 DEBUG_ASSERT(is_valid_vaddr(aspace, vaddr));
180 if (!is_valid_vaddr(aspace, vaddr))
181 return ERR_OUT_OF_RANGE;
182
183 vaddr = adjusted_vaddr(aspace, vaddr);
184 /* compute shift values based on if this address space is for kernel or user space */
185 if (aspace->flags & ARCH_ASPACE_FLAG_KERNEL) {
186 index_shift = MMU_KERNEL_TOP_SHIFT;
187 page_size_shift = MMU_KERNEL_PAGE_SIZE_SHIFT;
188
189 vaddr_t kernel_base = ~0UL << MMU_KERNEL_SIZE_SHIFT;
190 vaddr_rem = vaddr - kernel_base;
191
192 index = vaddr_rem >> index_shift;
193 ASSERT(index < MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP);
194 } else {
195 index_shift = MMU_USER_TOP_SHIFT;
196 page_size_shift = MMU_USER_PAGE_SIZE_SHIFT;
197
198 vaddr_rem = vaddr;
199 index = vaddr_rem >> index_shift;
200 ASSERT(index < MMU_USER_PAGE_TABLE_ENTRIES_TOP);
201 }
202
203 page_table = aspace->tt_virt;
204
205 while (true) {
206 index = vaddr_rem >> index_shift;
207 vaddr_rem -= (vaddr_t)index << index_shift;
208 pte = page_table[index];
209 descriptor_type = pte & MMU_PTE_DESCRIPTOR_MASK;
210 pte_addr = pte & MMU_PTE_OUTPUT_ADDR_MASK;
211
212 LTRACEF("va 0x%lx, index %d, index_shift %d, rem 0x%lx, pte 0x%" PRIx64 "\n",
213 vaddr, index, index_shift, vaddr_rem, pte);
214
215 if (descriptor_type == MMU_PTE_DESCRIPTOR_INVALID)
216 return ERR_NOT_FOUND;
217
218 if (descriptor_type == ((index_shift > page_size_shift) ?
219 MMU_PTE_L012_DESCRIPTOR_BLOCK :
220 MMU_PTE_L3_DESCRIPTOR_PAGE)) {
221 break;
222 }
223
224 if (index_shift <= page_size_shift ||
225 descriptor_type != MMU_PTE_L012_DESCRIPTOR_TABLE) {
226 PANIC_UNIMPLEMENTED;
227 }
228
229 page_table = paddr_to_kvaddr(pte_addr);
230 index_shift -= page_size_shift - 3;
231 }
232
233 if (paddr)
234 *paddr = pte_addr + vaddr_rem;
235 if (flags) {
236 uint mmu_flags = 0;
237 if (pte & MMU_PTE_ATTR_NON_SECURE)
238 mmu_flags |= ARCH_MMU_FLAG_NS;
239 switch (pte & MMU_PTE_ATTR_ATTR_INDEX_MASK) {
240 case MMU_PTE_ATTR_STRONGLY_ORDERED:
241 mmu_flags |= ARCH_MMU_FLAG_UNCACHED;
242 break;
243 case MMU_PTE_ATTR_DEVICE:
244 mmu_flags |= ARCH_MMU_FLAG_UNCACHED_DEVICE;
245 break;
246 case MMU_PTE_ATTR_NORMAL_MEMORY_TAGGED:
247 mmu_flags |= ARCH_MMU_FLAG_TAGGED;
248 break;
249 case MMU_PTE_ATTR_NORMAL_MEMORY:
250 mmu_flags |= ARCH_MMU_FLAG_CACHED;
251 break;
252 default:
253 PANIC_UNIMPLEMENTED;
254 }
255 switch (pte & MMU_PTE_ATTR_AP_MASK) {
256 case MMU_PTE_ATTR_AP_P_RW_U_NA:
257 break;
258 case MMU_PTE_ATTR_AP_P_RW_U_RW:
259 mmu_flags |= ARCH_MMU_FLAG_PERM_USER;
260 break;
261 case MMU_PTE_ATTR_AP_P_RO_U_NA:
262 mmu_flags |= ARCH_MMU_FLAG_PERM_RO;
263 break;
264 case MMU_PTE_ATTR_AP_P_RO_U_RO:
265 mmu_flags |= ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_RO;
266 break;
267 }
268 /*
269 * Based on whether or not this is a user page, check UXN or PXN
270 * bit to determine if it's an executable page.
271 */
272 if (mmu_flags & ARCH_MMU_FLAG_PERM_USER) {
273 DEBUG_ASSERT(pte & MMU_PTE_ATTR_PXN);
274 if (pte & MMU_PTE_ATTR_UXN) {
275 mmu_flags |= ARCH_MMU_FLAG_PERM_NO_EXECUTE;
276 }
277 } else {
278 DEBUG_ASSERT(pte & MMU_PTE_ATTR_UXN);
279 if (pte & MMU_PTE_ATTR_PXN) {
280 /* Privileged page, check the PXN bit. */
281 mmu_flags |= ARCH_MMU_FLAG_PERM_NO_EXECUTE;
282 }
283 }
284 *flags = mmu_flags;
285 }
286 LTRACEF("va 0x%lx, paddr 0x%lx, flags 0x%x\n",
287 vaddr, paddr ? *paddr : ~0UL, flags ? *flags : ~0U);
288 return 0;
289 }
290
alloc_page_table(paddr_t * paddrp,uint page_size_shift)291 static int alloc_page_table(paddr_t *paddrp, uint page_size_shift)
292 {
293 size_t size = 1U << page_size_shift;
294
295 LTRACEF("page_size_shift %u\n", page_size_shift);
296
297 if (size >= PAGE_SIZE) {
298 size_t count = size / PAGE_SIZE;
299 size_t ret = pmm_alloc_contiguous(count, page_size_shift, paddrp, NULL);
300 if (ret != count)
301 return ERR_NO_MEMORY;
302 } else {
303 void *vaddr = memalign(size, size);
304 if (!vaddr)
305 return ERR_NO_MEMORY;
306 *paddrp = vaddr_to_paddr(vaddr);
307 if (*paddrp == 0) {
308 free(vaddr);
309 return ERR_NO_MEMORY;
310 }
311 }
312
313 LTRACEF("allocated 0x%lx\n", *paddrp);
314 return 0;
315 }
316
free_page_table(void * vaddr,paddr_t paddr,uint page_size_shift)317 static void free_page_table(void *vaddr, paddr_t paddr, uint page_size_shift)
318 {
319 LTRACEF("vaddr %p paddr 0x%lx page_size_shift %u\n", vaddr, paddr, page_size_shift);
320
321 size_t size = 1U << page_size_shift;
322 vm_page_t *page;
323
324 if (size >= PAGE_SIZE) {
325 page = paddr_to_vm_page(paddr);
326 if (!page)
327 panic("bad page table paddr 0x%lx\n", paddr);
328 pmm_free_page(page);
329 } else {
330 free(vaddr);
331 }
332 }
333 #endif /* EARLY_MMU */
334
arm64_mmu_get_page_table(vaddr_t index,uint page_size_shift,pte_t * page_table)335 static pte_t *arm64_mmu_get_page_table(vaddr_t index, uint page_size_shift, pte_t *page_table)
336 {
337 pte_t pte;
338 paddr_t paddr;
339 void *vaddr;
340 int ret;
341
342 pte = page_table[index];
343 switch (pte & MMU_PTE_DESCRIPTOR_MASK) {
344 case MMU_PTE_DESCRIPTOR_INVALID:
345 ret = alloc_page_table(&paddr, page_size_shift);
346 if (ret) {
347 TRACEF("failed to allocate page table, page_size_shift %u\n", page_size_shift);
348 return NULL;
349 }
350 vaddr = paddr_to_kvaddr(paddr);
351
352 LTRACEF("allocated page table, vaddr %p, paddr 0x%lx\n", vaddr, paddr);
353 memset(vaddr, MMU_PTE_DESCRIPTOR_INVALID, 1U << page_size_shift);
354
355 __asm__ volatile("dmb ishst" ::: "memory");
356
357 pte = paddr | MMU_PTE_L012_DESCRIPTOR_TABLE;
358 page_table[index] = pte;
359 LTRACEF("pte %p[0x%lx] = 0x%" PRIx64 "\n", page_table, index, pte);
360 return vaddr;
361
362 case MMU_PTE_L012_DESCRIPTOR_TABLE:
363 paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK;
364 LTRACEF("found page table 0x%lx\n", paddr);
365 return paddr_to_kvaddr(paddr);
366
367 case MMU_PTE_L012_DESCRIPTOR_BLOCK:
368 return NULL;
369
370 default:
371 PANIC_UNIMPLEMENTED;
372 }
373 }
374
page_table_is_clear(pte_t * page_table,uint page_size_shift)375 static bool page_table_is_clear(pte_t *page_table, uint page_size_shift)
376 {
377 int i;
378 int count = 1U << (page_size_shift - 3);
379 pte_t pte;
380
381 for (i = 0; i < count; i++) {
382 pte = page_table[i];
383 if (pte != MMU_PTE_DESCRIPTOR_INVALID) {
384 LTRACEF("page_table at %p still in use, index %d is 0x%" PRIx64 "\n",
385 page_table, i, pte);
386 return false;
387 }
388 }
389
390 LTRACEF("page table at %p is clear\n", page_table);
391 return true;
392 }
393
arm64_mmu_unmap_pt(vaddr_t vaddr,vaddr_t vaddr_rel,size_t size,uint index_shift,uint page_size_shift,pte_t * page_table,uint asid)394 static void arm64_mmu_unmap_pt(vaddr_t vaddr, vaddr_t vaddr_rel,
395 size_t size,
396 uint index_shift, uint page_size_shift,
397 pte_t *page_table, uint asid)
398 {
399 pte_t *next_page_table;
400 vaddr_t index;
401 size_t chunk_size;
402 vaddr_t vaddr_rem;
403 vaddr_t block_size;
404 vaddr_t block_mask;
405 pte_t pte;
406 paddr_t page_table_paddr;
407
408 LTRACEF("vaddr 0x%lx, vaddr_rel 0x%lx, size 0x%lx, index shift %d, page_size_shift %d, page_table %p\n",
409 vaddr, vaddr_rel, size, index_shift, page_size_shift, page_table);
410
411 while (size) {
412 block_size = 1UL << index_shift;
413 block_mask = block_size - 1;
414 vaddr_rem = vaddr_rel & block_mask;
415 chunk_size = MIN(size, block_size - vaddr_rem);
416 index = vaddr_rel >> index_shift;
417
418 pte = page_table[index];
419
420 if (index_shift > page_size_shift &&
421 (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_TABLE) {
422 page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK;
423 next_page_table = paddr_to_kvaddr(page_table_paddr);
424 arm64_mmu_unmap_pt(vaddr, vaddr_rem, chunk_size,
425 index_shift - (page_size_shift - 3),
426 page_size_shift,
427 next_page_table, asid);
428 if (chunk_size == block_size ||
429 page_table_is_clear(next_page_table, page_size_shift)) {
430 LTRACEF("pte %p[0x%lx] = 0 (was page table)\n", page_table, index);
431 page_table[index] = MMU_PTE_DESCRIPTOR_INVALID;
432 __asm__ volatile("dmb ishst" ::: "memory");
433 free_page_table(next_page_table, page_table_paddr, page_size_shift);
434 }
435 } else if (pte) {
436 LTRACEF("pte %p[0x%lx] = 0\n", page_table, index);
437 page_table[index] = MMU_PTE_DESCRIPTOR_INVALID;
438 CF;
439 if (asid == MMU_ARM64_GLOBAL_ASID)
440 ARM64_TLBI(vaae1is, vaddr >> 12);
441 else
442 ARM64_TLBI(vae1is, vaddr >> 12 | (vaddr_t)asid << 48);
443 } else {
444 LTRACEF("pte %p[0x%lx] already clear\n", page_table, index);
445 }
446 size -= chunk_size;
447 if (!size) {
448 break;
449 }
450 /* Early out avoids a benign overflow. */
451 vaddr += chunk_size;
452 vaddr_rel += chunk_size;
453 }
454 }
455
arm64_mmu_map_pt(vaddr_t vaddr_in,vaddr_t vaddr_rel_in,paddr_t paddr_in,size_t size_in,pte_t attrs,uint index_shift,uint page_size_shift,pte_t * page_table,uint asid,bool replace)456 static int arm64_mmu_map_pt(vaddr_t vaddr_in, vaddr_t vaddr_rel_in,
457 paddr_t paddr_in,
458 size_t size_in, pte_t attrs,
459 uint index_shift, uint page_size_shift,
460 pte_t *page_table, uint asid, bool replace)
461 {
462 int ret;
463 pte_t *next_page_table;
464 vaddr_t index;
465 vaddr_t vaddr = vaddr_in;
466 vaddr_t vaddr_rel = vaddr_rel_in;
467 paddr_t paddr = paddr_in;
468 size_t size = size_in;
469 size_t chunk_size;
470 vaddr_t vaddr_rem;
471 vaddr_t block_size;
472 vaddr_t block_mask;
473 pte_t pte;
474
475 LTRACEF("vaddr 0x%lx, vaddr_rel 0x%lx, paddr 0x%lx, size 0x%lx, attrs 0x%" PRIx64 ", index shift %d, page_size_shift %d, page_table %p\n",
476 vaddr, vaddr_rel, paddr, size, attrs,
477 index_shift, page_size_shift, page_table);
478
479 if ((vaddr_rel | paddr | size) & ((1UL << page_size_shift) - 1)) {
480 TRACEF("not page aligned\n");
481 return ERR_INVALID_ARGS;
482 }
483
484 while (size) {
485 block_size = 1UL << index_shift;
486 block_mask = block_size - 1;
487 vaddr_rem = vaddr_rel & block_mask;
488 chunk_size = MIN(size, block_size - vaddr_rem);
489 index = vaddr_rel >> index_shift;
490
491 if (((vaddr_rel | paddr) & block_mask) ||
492 (chunk_size != block_size) ||
493 (index_shift > MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT)) {
494 next_page_table = arm64_mmu_get_page_table(index, page_size_shift,
495 page_table);
496 if (!next_page_table)
497 goto err;
498
499 ret = arm64_mmu_map_pt(vaddr, vaddr_rem, paddr, chunk_size, attrs,
500 index_shift - (page_size_shift - 3),
501 page_size_shift, next_page_table, asid,
502 replace);
503 if (ret)
504 goto err;
505 } else {
506 pte = page_table[index];
507 if (!pte && replace) {
508 TRACEF("page table entry does not exist yet, index 0x%lx, 0x%" PRIx64 "\n",
509 index, pte);
510 goto err;
511 }
512 if (pte && !replace) {
513 TRACEF("page table entry already in use, index 0x%lx, 0x%" PRIx64 "\n",
514 index, pte);
515 goto err;
516 }
517
518 pte = paddr | attrs;
519 if (index_shift > page_size_shift)
520 pte |= MMU_PTE_L012_DESCRIPTOR_BLOCK;
521 else
522 pte |= MMU_PTE_L3_DESCRIPTOR_PAGE;
523
524 LTRACEF("pte %p[0x%lx] = 0x%" PRIx64 "\n", page_table, index, pte);
525 page_table[index] = pte;
526 if (replace) {
527 CF;
528 if (asid == MMU_ARM64_GLOBAL_ASID)
529 ARM64_TLBI(vaae1is, vaddr >> 12);
530 else
531 ARM64_TLBI(vae1is, vaddr >> 12 | (vaddr_t)asid << 48);
532 }
533 }
534 size -= chunk_size;
535 if (!size) {
536 break;
537 }
538 /* Note: early out avoids a benign overflow. */
539 vaddr += chunk_size;
540 vaddr_rel += chunk_size;
541 paddr += chunk_size;
542 }
543
544 return 0;
545
546 err:
547 arm64_mmu_unmap_pt(vaddr_in, vaddr_rel_in, size_in - size,
548 index_shift, page_size_shift, page_table, asid);
549 DSB;
550 return ERR_GENERIC;
551 }
552
553 #ifndef EARLY_MMU
arm64_mmu_map(vaddr_t vaddr,paddr_t paddr,size_t size,pte_t attrs,vaddr_t vaddr_base,uint top_size_shift,uint top_index_shift,uint page_size_shift,pte_t * top_page_table,uint asid,bool replace)554 int arm64_mmu_map(vaddr_t vaddr, paddr_t paddr, size_t size, pte_t attrs,
555 vaddr_t vaddr_base, uint top_size_shift,
556 uint top_index_shift, uint page_size_shift,
557 pte_t *top_page_table, uint asid, bool replace)
558 {
559 int ret;
560 vaddr_t vaddr_rel = vaddr - vaddr_base;
561 vaddr_t vaddr_rel_max = 1UL << top_size_shift;
562
563 LTRACEF("vaddr 0x%lx, paddr 0x%lx, size 0x%lx, attrs 0x%" PRIx64 ", asid 0x%x\n",
564 vaddr, paddr, size, attrs, asid);
565
566 if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) {
567 TRACEF("vaddr 0x%lx, size 0x%lx out of range vaddr 0x%lx, size 0x%lx\n",
568 vaddr, size, vaddr_base, vaddr_rel_max);
569 return ERR_INVALID_ARGS;
570 }
571
572 if (!top_page_table) {
573 TRACEF("page table is NULL\n");
574 return ERR_INVALID_ARGS;
575 }
576
577 ret = arm64_mmu_map_pt(vaddr, vaddr_rel, paddr, size, attrs,
578 top_index_shift, page_size_shift, top_page_table,
579 asid, replace);
580 DSB;
581 return ret;
582 }
583
arm64_mmu_unmap(vaddr_t vaddr,size_t size,vaddr_t vaddr_base,uint top_size_shift,uint top_index_shift,uint page_size_shift,pte_t * top_page_table,uint asid)584 int arm64_mmu_unmap(vaddr_t vaddr, size_t size,
585 vaddr_t vaddr_base, uint top_size_shift,
586 uint top_index_shift, uint page_size_shift,
587 pte_t *top_page_table, uint asid)
588 {
589 vaddr_t vaddr_rel = vaddr - vaddr_base;
590 vaddr_t vaddr_rel_max = 1UL << top_size_shift;
591
592 LTRACEF("vaddr 0x%lx, size 0x%lx, asid 0x%x\n", vaddr, size, asid);
593
594 if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) {
595 TRACEF("vaddr 0x%lx, size 0x%lx out of range vaddr 0x%lx, size 0x%lx\n",
596 vaddr, size, vaddr_base, vaddr_rel_max);
597 return ERR_INVALID_ARGS;
598 }
599
600 if (!top_page_table) {
601 TRACEF("page table is NULL\n");
602 return ERR_INVALID_ARGS;
603 }
604
605 arm64_mmu_unmap_pt(vaddr, vaddr_rel, size,
606 top_index_shift, page_size_shift, top_page_table, asid);
607 DSB;
608 return 0;
609 }
610
arm64_tlbflush_if_asid_changed(arch_aspace_t * aspace,asid_t asid)611 static void arm64_tlbflush_if_asid_changed(arch_aspace_t *aspace, asid_t asid)
612 {
613 THREAD_LOCK(state);
614 if (asid != arch_mmu_asid(aspace)) {
615 TRACEF("asid changed for aspace %p while mapping or unmapping memory, 0x%" PRIxASID " -> 0x%" PRIxASID ", flush all tlbs\n",
616 aspace, asid, aspace->asid);
617 ARM64_TLBI_NOADDR(vmalle1is);
618 DSB;
619 }
620 THREAD_UNLOCK(state);
621 }
622
arm64_mmu_map_aspace(arch_aspace_t * aspace,vaddr_t vaddr,paddr_t paddr,size_t count,uint flags,bool replace)623 static int arm64_mmu_map_aspace(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t paddr, size_t count, uint flags,
624 bool replace)
625 {
626 LTRACEF("vaddr 0x%lx paddr 0x%lx count %zu flags 0x%x\n", vaddr, paddr, count, flags);
627
628 DEBUG_ASSERT(aspace);
629 DEBUG_ASSERT(aspace->tt_virt);
630
631 DEBUG_ASSERT(is_valid_vaddr(aspace, vaddr));
632 if (!is_valid_vaddr(aspace, vaddr))
633 return ERR_OUT_OF_RANGE;
634
635 vaddr = adjusted_vaddr(aspace, vaddr);
636
637 /* paddr and vaddr must be aligned */
638 DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr));
639 DEBUG_ASSERT(IS_PAGE_ALIGNED(paddr));
640 if (!IS_PAGE_ALIGNED(vaddr) || !IS_PAGE_ALIGNED(paddr))
641 return ERR_INVALID_ARGS;
642
643 if (paddr & ~MMU_PTE_OUTPUT_ADDR_MASK) {
644 return ERR_INVALID_ARGS;
645 }
646
647 if (count == 0)
648 return NO_ERROR;
649
650 pte_t pte_attr;
651 if (!mmu_flags_to_pte_attr(aspace->flags, flags, &pte_attr)) {
652 return ERR_INVALID_ARGS;
653 }
654
655 int ret;
656 if (aspace->flags & ARCH_ASPACE_FLAG_KERNEL) {
657 ret = arm64_mmu_map(vaddr, paddr, count * PAGE_SIZE,
658 pte_attr,
659 ~0UL << MMU_KERNEL_SIZE_SHIFT, MMU_KERNEL_SIZE_SHIFT,
660 MMU_KERNEL_TOP_SHIFT, MMU_KERNEL_PAGE_SIZE_SHIFT,
661 aspace->tt_virt, MMU_ARM64_GLOBAL_ASID, replace);
662 } else {
663 asid_t asid = arch_mmu_asid(aspace);
664 ret = arm64_mmu_map(vaddr, paddr, count * PAGE_SIZE,
665 pte_attr | MMU_PTE_ATTR_NON_GLOBAL,
666 0, MMU_USER_SIZE_SHIFT,
667 MMU_USER_TOP_SHIFT, MMU_USER_PAGE_SIZE_SHIFT,
668 aspace->tt_virt, asid, replace);
669 arm64_tlbflush_if_asid_changed(aspace, asid);
670 }
671
672 return ret;
673 }
674
arch_mmu_map(arch_aspace_t * aspace,vaddr_t vaddr,paddr_t paddr,size_t count,uint flags)675 int arch_mmu_map(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t paddr, size_t count, uint flags)
676 {
677 return arm64_mmu_map_aspace(aspace, vaddr, paddr, count, flags, false);
678 }
679
arch_mmu_map_replace(arch_aspace_t * aspace,vaddr_t vaddr,paddr_t paddr,size_t count,uint flags)680 int arch_mmu_map_replace(arch_aspace_t *aspace, vaddr_t vaddr, paddr_t paddr, size_t count, uint flags)
681 {
682 return arm64_mmu_map_aspace(aspace, vaddr, paddr, count, flags, true);
683 }
684
arch_mmu_unmap(arch_aspace_t * aspace,vaddr_t vaddr,size_t count)685 int arch_mmu_unmap(arch_aspace_t *aspace, vaddr_t vaddr, size_t count)
686 {
687 LTRACEF("vaddr 0x%lx count %zu\n", vaddr, count);
688
689 DEBUG_ASSERT(aspace);
690 DEBUG_ASSERT(aspace->tt_virt);
691
692 DEBUG_ASSERT(is_valid_vaddr(aspace, vaddr));
693
694 if (!is_valid_vaddr(aspace, vaddr))
695 return ERR_OUT_OF_RANGE;
696
697 vaddr = adjusted_vaddr(aspace, vaddr);
698
699 DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr));
700 if (!IS_PAGE_ALIGNED(vaddr))
701 return ERR_INVALID_ARGS;
702
703 int ret;
704 if (aspace->flags & ARCH_ASPACE_FLAG_KERNEL) {
705 ret = arm64_mmu_unmap(vaddr, count * PAGE_SIZE,
706 ~0UL << MMU_KERNEL_SIZE_SHIFT, MMU_KERNEL_SIZE_SHIFT,
707 MMU_KERNEL_TOP_SHIFT, MMU_KERNEL_PAGE_SIZE_SHIFT,
708 aspace->tt_virt,
709 MMU_ARM64_GLOBAL_ASID);
710 } else {
711 asid_t asid = arch_mmu_asid(aspace);
712 ret = arm64_mmu_unmap(vaddr, count * PAGE_SIZE,
713 0, MMU_USER_SIZE_SHIFT,
714 MMU_USER_TOP_SHIFT, MMU_USER_PAGE_SIZE_SHIFT,
715 aspace->tt_virt, asid);
716 arm64_tlbflush_if_asid_changed(aspace, asid);
717 }
718
719 return ret;
720 }
721
arch_mmu_init_aspace(arch_aspace_t * aspace,vaddr_t base,size_t size,uint flags)722 status_t arch_mmu_init_aspace(arch_aspace_t *aspace, vaddr_t base, size_t size, uint flags)
723 {
724 LTRACEF("aspace %p, base 0x%lx, size 0x%zx, flags 0x%x\n", aspace, base, size, flags);
725
726 DEBUG_ASSERT(aspace);
727
728 /* validate that the base + size is sane and doesn't wrap */
729 DEBUG_ASSERT(size > PAGE_SIZE);
730 DEBUG_ASSERT(wrap_check(base, size));
731
732 aspace->flags = flags;
733 if (flags & ARCH_ASPACE_FLAG_KERNEL) {
734 /* at the moment we can only deal with address spaces as globally defined */
735 DEBUG_ASSERT(base == ~0UL << MMU_KERNEL_SIZE_SHIFT);
736 DEBUG_ASSERT(size == 1UL << MMU_KERNEL_SIZE_SHIFT);
737
738 aspace->base = base;
739 aspace->size = size;
740 aspace->tt_virt = arm64_kernel_translation_table;
741 aspace->tt_phys = vaddr_to_paddr(aspace->tt_virt);
742 } else {
743 size_t page_table_size = MMU_USER_PAGE_TABLE_ENTRIES_TOP * sizeof(pte_t);
744 //DEBUG_ASSERT(base >= 0);
745 DEBUG_ASSERT(base + size <= 1UL << MMU_USER_SIZE_SHIFT);
746
747 aspace->base = base;
748 aspace->size = size;
749
750 pte_t *va = memalign(page_table_size, page_table_size);
751 if (!va)
752 return ERR_NO_MEMORY;
753
754 aspace->tt_virt = va;
755 aspace->tt_phys = vaddr_to_paddr(aspace->tt_virt);
756
757 /* zero the top level translation table */
758 memset(aspace->tt_virt, 0, page_table_size);
759 }
760
761 LTRACEF("tt_phys 0x%lx tt_virt %p\n", aspace->tt_phys, aspace->tt_virt);
762
763 return NO_ERROR;
764 }
765
arch_mmu_destroy_aspace(arch_aspace_t * aspace)766 status_t arch_mmu_destroy_aspace(arch_aspace_t *aspace)
767 {
768 LTRACEF("aspace %p\n", aspace);
769
770 DEBUG_ASSERT(aspace);
771 DEBUG_ASSERT((aspace->flags & ARCH_ASPACE_FLAG_KERNEL) == 0);
772
773 // XXX make sure it's not mapped
774
775 free(aspace->tt_virt);
776
777 return NO_ERROR;
778 }
779
arch_mmu_context_switch(arch_aspace_t * aspace)780 void arch_mmu_context_switch(arch_aspace_t *aspace)
781 {
782 bool flush_tlb;
783
784 if (TRACE_CONTEXT_SWITCH)
785 TRACEF("aspace %p\n", aspace);
786
787 flush_tlb = vmm_asid_activate(aspace, ARM64_ASID_BITS);
788
789 uint64_t tcr;
790 uint64_t ttbr;
791 if (aspace) {
792 DEBUG_ASSERT((aspace->flags & ARCH_ASPACE_FLAG_KERNEL) == 0);
793
794 tcr = MMU_TCR_FLAGS_USER;
795 ttbr = (arch_mmu_asid(aspace) << 48) | aspace->tt_phys;
796 ARM64_WRITE_SYSREG(ttbr0_el1, ttbr);
797
798 } else {
799 tcr = MMU_TCR_FLAGS_KERNEL;
800 }
801
802 #if KERNEL_PAC_ENABLED
803 /* If TBI0 is set, also set TBID0.
804 * TBID0 is RES0 if FEAT_PAuth is not supported, so this must be
805 * conditional.
806 */
807 if ((tcr & MMU_TCR_TBI0) != 0 && arch_pac_address_supported()) {
808 tcr |= MMU_TCR_TBID0;
809 }
810 #endif
811
812 if (TRACE_CONTEXT_SWITCH) {
813 if (aspace) {
814 TRACEF("ttbr 0x%" PRIx64 ", tcr 0x%" PRIx64 "\n", ttbr, tcr);
815 } else {
816 TRACEF("tcr 0x%" PRIx64 "\n", tcr);
817 }
818 }
819
820 ARM64_WRITE_SYSREG(tcr_el1, tcr); /* TODO: only needed when switching between kernel and user threads */
821
822 if (flush_tlb) {
823 ARM64_TLBI_NOADDR(vmalle1);
824 DSB;
825 }
826 }
827 #endif /* EARLY_MMU */
828