1#include <asm.h>
2#include <arch/arm64/mmu.h>
3#include <arch/asm_macros.h>
4#include <kernel/vm.h>
5
6/*
7 * Register use:
8 *  x0-x3   Arguments
9 *  x9-x15  Scratch
10 *  x18     Shadow stack pointer (if enabled)
11 *  x19-x28 Globals
12 */
13tmp                     .req x9
14tmp2                    .req x10
15wtmp2                   .req w10
16index                   .req x11
17canary                  .req x12
18
19ssp                     .req x18
20cpuid                   .req x19
21page_table0             .req x20
22page_table1             .req x21
23size                    .req x22
24supports_mte            .req x23
25
26.section .text.boot
27.globl arm_reset
28arm_reset:
29.globl _start
30.type _start,STT_OBJECT
31_start:
32    /* This instruction is read by the bootloader to determine image type */
33    bl      arm64_elX_to_el1
34
35    /* Initialize VBAR to the temporary exception vector table */
36    adrl    tmp, .Learly_exception_base
37    msr     vbar_el1, tmp
38    isb
39
40    mrs     tmp2, id_aa64pfr1_el1
41    tst     tmp2, #0xe00
42    cset    supports_mte, ne
43
44#if WITH_KERNEL_VM
45    /* enable caches so atomics and spinlocks work */
46    mrs     tmp, sctlr_el1
47    orr     tmp, tmp, #(1<<12) /* Enable icache */
48    orr     tmp, tmp, #(1<<2)  /* Enable dcache/ucache */
49    orr     tmp, tmp, #(1<<3)  /* Enable Stack Alignment Check EL1 */
50    orr     tmp, tmp, #(1<<4)  /* Enable Stack Alignment Check EL0 */
51    cbz     supports_mte, .Ldont_set_mte_flags
52    orr     tmp, tmp, #(1<<43) /* Allocation Tag Access in EL1 */
53    orr     tmp, tmp, #(1<<42) /* Allocation Tag Access in EL0 */
54    bic     tmp, tmp, #(1<<40) /* No tag check faults in EL1 */
55    orr     tmp, tmp, #(1<<38) /* Tag check faults in EL0 are synchronous */
56.Ldont_set_mte_flags:
57    bic     tmp, tmp, #(1<<1)  /* Disable Alignment Checking for EL1 EL0 */
58    msr     sctlr_el1, tmp
59
60    /* set up the mmu according to mmu_initial_mappings */
61
62    /* load the base of the translation table and clear the table */
63    adrl    page_table1, arm64_kernel_translation_table
64
65    /* Prepare tt_trampoline page table */
66    /* Calculate pagetable physical addresses */
67    adrl    page_table0, tt_trampoline
68
69#if WITH_SMP
70    /*
71     * Stash x0 as it will be clobbered
72     * We place it in size as x0 contains the size passed to the entry point.
73     */
74    mov     size, x0
75    /* Get the CPU number */
76    bl      arm64_curr_cpu_num
77    mov     cpuid, x0
78    /* Restore registers */
79    mov     x0, size
80    cbnz    cpuid, .Ltt_trampoline_check_secondary
81#endif
82
83    /* Zero the top level kernel page table */
84    mov     tmp, #0
85
86    /* walk through all the entries in the translation table, setting them up */
87.Lclear_top_page_table_loop:
88    str     xzr, [page_table1, tmp, lsl #3]
89    add     tmp, tmp, #1
90    cmp     tmp, #MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP
91    bne     .Lclear_top_page_table_loop
92
93    /* Prepare tt_trampoline page table */
94
95    /* Zero tt_trampoline translation tables */
96    mov     tmp, #0
97.Lclear_tt_trampoline:
98    str     xzr, [page_table0, tmp, lsl#3]
99    add     tmp, tmp, #1
100    cmp     tmp, #MMU_PAGE_TABLE_ENTRIES_IDENT
101    blt     .Lclear_tt_trampoline
102
103    /* Setup mapping at phys -> phys */
104    /*
105     * Map from the start of the kernel to the end of RAM
106     * so we have enough pages for boot_alloc memory.
107     */
108    adr     index, _start
109    lsr     tmp, index, #MMU_IDENT_TOP_SHIFT    /* tmp = paddr index */
110
111    /* Check that the start index falls inside the table */
112    cmp     tmp, #MMU_PAGE_TABLE_ENTRIES_IDENT
113    b.hs    platform_early_halt
114
115#if ARM64_BOOT_PROTOCOL_X0_MEMSIZE
116    /*
117     * The physical address of end of ram (exclusive) is (_start + x0).
118     */
119    add     index, index, x0
120#elif ARM64_BOOT_PROTOCOL_X0_DTB
121    /*
122     * The physical address if end of kernel (exclusive) that could be used
123     * before any dynamic memory allocations are made is &_end.
124     * (The rest will be mapped in arm64_early_mmu_init)
125     */
126    adrl    index, _end
127#else
128    #error Unknown ARM64_BOOT_PROTOCOL
129#endif
130    /*
131     * `index` is the first byte of memory after the kernel that we don't need
132     * mapped at this point. We subtract one and round that down to a multiple
133     * of 1<<MMU_IDENT_TOP_SHIFT to get the inclusive upper bound of the
134     * tt_trampoline range.
135     */
136    sub     index, index, #1
137    lsr     index, index, #MMU_IDENT_TOP_SHIFT
138
139    /* Clamp the end index to the last possible entry */
140    cmp     index, #MMU_PAGE_TABLE_ENTRIES_IDENT
141    b.lo    .Lset_tt_trampoline_loop
142    mov     index, #(MMU_PAGE_TABLE_ENTRIES_IDENT - 1)
143
144.Lset_tt_trampoline_loop:
145    cmp     tmp, index
146    b.hi   .Lset_tt_trampoline_done
147
148    ldr     tmp2, =MMU_PTE_IDENT_FLAGS
149    cbz     supports_mte, .Luse_untagged_mapping
150    ldr     tmp2, =MMU_PTE_IDENT_FLAGS_TAGGED
151.Luse_untagged_mapping:
152    add     tmp2, tmp2, tmp, lsl #MMU_IDENT_TOP_SHIFT  /* tmp2 = pt entry */
153    str     tmp2, [page_table0, tmp, lsl #3]     /* tt_trampoline[paddr index] = pt entry */
154    add     tmp, tmp, #1
155    b       .Lset_tt_trampoline_loop
156
157.Lset_tt_trampoline_done:
158
159#if WITH_SMP
160    /* Release the first lock on the secondary CPUs */
161    adrl    tmp, tt_trampoline_not_ready
162    str     wzr, [tmp]
163    b       .Ltt_trampoline_ready
164
165.Ltt_trampoline_check_secondary:
166    adrl    tmp, tt_trampoline_not_ready
167.Ltt_trampoline_not_ready:
168    ldr     wtmp2, [tmp]
169    cbnz    wtmp2, .Ltt_trampoline_not_ready
170.Ltt_trampoline_ready:
171#endif
172
173    /* set up the mmu */
174
175    /* Invalidate TLB */
176    tlbi    vmalle1is
177    isb
178    dsb     sy
179
180    /* Initialize Memory Attribute Indirection Register */
181    ldr     tmp, =MMU_MAIR_VAL
182    msr     mair_el1, tmp
183
184    /* Initialize TCR_EL1 */
185    /* set cacheable attributes on translation walk */
186    /* (SMP extensions) non-shareable, inner write-back write-allocate */
187    ldr     tmp, =MMU_TCR_FLAGS_IDENT
188    msr     tcr_el1, tmp
189
190    isb
191
192    /* Write ttbr with phys addr of the translation table */
193    msr     ttbr0_el1, page_table0
194    msr     ttbr1_el1, page_table1
195    isb
196
197    /* Read SCTLR */
198    mrs     tmp, sctlr_el1
199
200    /* Turn on the MMU */
201    orr     tmp, tmp, #0x1
202
203    /* Write back SCTLR */
204    msr     sctlr_el1, tmp
205    isb
206
207#if WITH_SMP
208    cbnz    cpuid, .Lpage_tables_check_secondary
209#endif
210#endif /* WITH_KERNEL_VM */
211
212    /* clear bss */
213.L__do_bss:
214    /* clear out the bss excluding the stack and kernel translation table  */
215    /* NOTE: relies on __post_prebss_bss_start and __bss_end being 8 byte aligned */
216    adrl    tmp, __post_prebss_bss_start
217    adrl    tmp2, __bss_end
218    sub     tmp2, tmp2, tmp
219    cbz     tmp2, .L__bss_loop_done
220.L__bss_loop:
221    sub     tmp2, tmp2, #8
222    str     xzr, [tmp], #8
223    cbnz    tmp2, .L__bss_loop
224.L__bss_loop_done:
225
226#if WITH_KERNEL_VM
227    /* Set up the stack */
228    adrl    tmp, __stack_end
229    mov     sp, tmp
230
231    /* Add the stack canary region at the low end of the stack */
232    ldr     tmp2, =ARCH_DEFAULT_STACK_SIZE
233    sub     tmp, tmp, tmp2
234    ldr     index, =ARM64_PHYSICAL_STACK_CANARY_WORDS
235    ldr     canary, =ARM64_PHYSICAL_STACK_CANARY
236
237.Lphysical_stack_canary_setup_loop:
238    cbz     index, .Lphysical_stack_canary_setup_end
239
240    /* Store the canary at the current stack location */
241    str     canary, [tmp], #8
242
243    sub     index, index, #1
244    /* Rotate the canary so every value is different */
245    ror     canary, canary, #ARM64_PHYSICAL_STACK_CANARY_ROTATE
246    b       .Lphysical_stack_canary_setup_loop
247
248.Lphysical_stack_canary_setup_end:
249
250#if KERNEL_SCS_ENABLED
251    adrl    ssp, __shadow_stack
252#endif
253
254    /* Try to write and read tags to test if EL3 configured tag space access */
255    cbz     supports_mte, .Lno_tagging_feature
256
257    /*
258     * The stack is set up but unused at this point, so we can use
259     * it as a convenient address for tagging.
260     */
261    mov     tmp, sp
262    sub     tmp, tmp, #16
263    mov     tmp2, #3
264    bfi     tmp, tmp2, #56, #4
265    mov     tmp2, tmp
266.arch_extension memtag
267    stg     tmp, [tmp]
268    ldg     tmp, [tmp]
269    cmp     tmp, tmp2
270    cset    tmp2, eq
271    adrl    tmp, arm64_mte_enabled
272    strb    wtmp2, [tmp]
273
274.Lno_tagging_feature:
275
276    /* Save the arguments */
277    push    x2, x3
278    push    x0, x1
279
280    /* x0 already contains ram_size */
281    adrl    x1, __relr_start
282    adrl    x2, __relr_end
283    adrl    x3, _start
284    bl      arm64_early_mmu_init
285
286    /* Check the stack canaries */
287    adrl    tmp, __stack_end
288    ldr     tmp2, =ARCH_DEFAULT_STACK_SIZE
289    sub     tmp, tmp, tmp2
290    ldr     index, =ARM64_PHYSICAL_STACK_CANARY_WORDS
291    ldr     canary, =ARM64_PHYSICAL_STACK_CANARY
292
293.Lphysical_stack_canary_check_loop:
294    cbz     index, .Lphysical_stack_canary_check_end
295
296    ldr     tmp2, [tmp], #8
297    cmp     tmp2, canary
298    b.ne    platform_early_halt /* Error: canary got overwritten, stack overflow */
299
300    sub     index, index, #1
301    /* Rotate the canary so every value is different */
302    ror     canary, canary, #ARM64_PHYSICAL_STACK_CANARY_ROTATE
303    b       .Lphysical_stack_canary_check_loop
304
305.Lphysical_stack_canary_check_end:
306
307    /* Restore the arguments */
308    pop     x0, x1
309    pop     x2, x3
310
311    /* Check the stack offset */
312    adrl    tmp, __stack_end
313    mov     tmp2, sp
314    cmp     tmp, tmp2
315    b.ne    platform_early_halt /* Error: invalid SP on return from C */
316
317#if WITH_SMP
318    /* Release the second lock on the secondary CPUs */
319    adrl    tmp, page_tables_not_ready
320    stlr    wzr, [tmp]
321    b       .Lpage_tables_ready
322
323.Lpage_tables_check_secondary:
324    adrl    tmp, page_tables_not_ready
325.Lpage_tables_not_ready:
326    ldar    wtmp2, [tmp]
327    cbnz    wtmp2, .Lpage_tables_not_ready
328.Lpage_tables_ready:
329#endif
330    isb
331
332    /* Jump to virtual code address */
333    adrl    tmp, mmu_on_vaddr_ptr
334    ldr     tmp, [tmp]
335    br      tmp
336
337.Lmmu_on_vaddr:
338    /* Allow br from above if BTI is enabled, otherwise this is a NOP */
339    bti     j
340
341    /* Update VBAR to its virtual address */
342    adrl    tmp, .Learly_exception_base
343    msr     vbar_el1, tmp
344    isb
345
346    /* Disable trampoline page-table in ttbr0 */
347    ldr     tmp, =MMU_TCR_FLAGS_KERNEL
348    msr     tcr_el1, tmp
349    isb
350
351    /* Enable Write implies XN (Execute-Never), EL0/1 */
352    mrs     tmp, sctlr_el1
353    orr     tmp, tmp, #(1<<19)
354    msr     sctlr_el1, tmp
355    isb
356
357    /* Invalidate TLB and sync (needed for at least WXN) */
358    tlbi    vmalle1
359    dsb     sy
360    isb
361
362    /* We're no longer using the tagged identity map at this point, so
363     * enable synchronous tag check faults in EL1 to catch any code
364     * improperly using a tagged mapping
365     */
366    cbz     supports_mte, .Ldont_set_mte_flags_2
367    mrs     tmp, sctlr_el1
368    orr     tmp, tmp, #(1<<40)
369    msr     sctlr_el1, tmp
370.Ldont_set_mte_flags_2:
371
372#if WITH_SMP
373    cbnz    cpuid, .Lsecondary_boot
374#endif
375#endif /* WITH_KERNEL_VM */
376
377    adrl    tmp, sp_el1_bufs
378    mov     sp, tmp
379
380    msr     spsel, #0 /* Use SP0 for kernel stacks */
381    adrl    tmp, __stack_end
382    mov sp, tmp
383
384#if KERNEL_SCS_ENABLED
385    adrl    ssp, __shadow_stack
386#endif
387
388    bl  lk_main
389    b   .
390
391#if WITH_SMP
392/*
393 *  Decodes the CPU number out of MPIDR.
394 *  This can be overridden by the platform.
395 *  If it is, it must:
396 *  - Not assume a stack
397 *  - Only clobber x0, x9, x10
398 *  - Return the CPU number in x0
399 *  - If the CPU number would be invalid, return SMP_MAX_CPUS
400 */
401WEAK_FUNCTION(arm64_curr_cpu_num)
402    mrs     x0, mpidr_el1
403    ubfx    x0, x0, #0, #SMP_CPU_ID_BITS
404    and     tmp, x0, #0xff
405    cmp     tmp, #(1 << SMP_CPU_CLUSTER_SHIFT)
406    bge     .Lunsupported_cpu_num
407    bic     x0, x0, #0xff
408    orr     x0, tmp, x0, LSR #(8 - SMP_CPU_CLUSTER_SHIFT)
409    ret
410
411.Lunsupported_cpu_num:
412    mov     x0, #SMP_MAX_CPUS
413    ret
414
415.Lsecondary_boot:
416    cmp     cpuid, #SMP_MAX_CPUS
417    bge     .Lunsupported_cpu_trap
418
419    /* Set up the stack pointers */
420    adrl    tmp, sp_el1_bufs
421    mov     tmp2, #ARM64_EXC_SP_EL1_BUF_SIZE
422    mul     tmp2, tmp2, cpuid
423    add     sp, tmp, tmp2
424
425    msr     spsel, #0 /* Use SP0 for kernel stacks */
426
427    adrl    tmp, __stack_end
428    mov     tmp2, #ARCH_DEFAULT_STACK_SIZE
429    mul     tmp2, tmp2, cpuid
430    sub     sp, tmp, tmp2
431
432#if KERNEL_SCS_ENABLED
433    /* Set up the shadow stack pointer */
434    adrl    ssp, __shadow_stack
435    mov     tmp, #ARCH_DEFAULT_SHADOW_STACK_SIZE
436    mul     tmp, tmp, cpuid
437    add     ssp, ssp, tmp
438#endif
439
440    mov     x0, cpuid
441    bl      arm64_secondary_entry
442
443.Lunsupported_cpu_trap:
444    wfe
445    b       .Lunsupported_cpu_trap
446#endif
447
448.ltorg
449
450.section .text.boot.early.vectab
451.balign 0x800 /* This is the required alignment for the table */
452.Learly_exception_base:
453.org 0x000
454    b platform_early_halt
455.org 0x080
456    b platform_early_halt
457.org 0x100
458    b platform_early_halt
459.org 0x180
460    b platform_early_halt
461.org 0x200
462    b platform_early_halt
463.org 0x280
464    b platform_early_halt
465.org 0x300
466    b platform_early_halt
467.org 0x380
468    b platform_early_halt
469.org 0x400
470    b platform_early_halt
471.org 0x480
472    b platform_early_halt
473.org 0x500
474    b platform_early_halt
475.org 0x580
476    b platform_early_halt
477.org 0x600
478    b platform_early_halt
479.org 0x680
480    b platform_early_halt
481.org 0x700
482    b platform_early_halt
483.org 0x780
484    b platform_early_halt
485
486#if WITH_KERNEL_VM
487.data
488DATA(mmu_on_vaddr_ptr)
489.hidden mmu_on_vaddr_ptr
490    /*
491     * Store a pointer to the virtual address of .Lmmu_on_vaddr inside a
492     * pointer quad that ASLR can relocate.
493     */
494    .align 3
495    .quad .Lmmu_on_vaddr
496#endif
497
498#if WITH_SMP
499.data
500DATA(tt_trampoline_not_ready)
501    /*
502     * The primary processor clears this when the ttbr0 page tables
503     * are ready and all processors can enable their MMUs. Before
504     * passing this semaphore, all CPUs should have MMUs off, and
505     * turn them on immediately after.
506     */
507    .long       1
508DATA(page_tables_not_ready)
509    /*
510     * The primary processor clears this when the final (ttbr1)
511     * page tables are ready.
512     */
513    .long       1
514#endif
515
516.section .bss.prebss.stack
517    .align 4
518DATA(__stack)
519    .skip ARCH_DEFAULT_STACK_SIZE * SMP_MAX_CPUS
520DATA(__stack_end)
521
522#if KERNEL_SCS_ENABLED
523.section .bss.prebss.shadow_stack
524    /*
525     * Request 2^3 = 8-byte alignment. For aarch64, the stack pointer
526     * alignment must be two times the pointer size (2^4) but the same is not
527     * required for the shadow stack. Protecting the shadow stack with Memory
528     * Tagging Extensions may require matching MTE's 16-byte tag granularity.
529     */
530    .align 3
531DATA(__shadow_stack)
532    .skip ARCH_DEFAULT_SHADOW_STACK_SIZE * SMP_MAX_CPUS
533#endif
534
535.section .bss.prebss.sp_el1_bufs
536    .align 4
537DATA(sp_el1_bufs)
538    .skip ARM64_EXC_SP_EL1_BUF_SIZE * SMP_MAX_CPUS
539
540#if WITH_KERNEL_VM
541.section ".bss.prebss.translation_table"
542.align 3 + MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT
543DATA(tt_trampoline)
544    .skip 8 * MMU_PAGE_TABLE_ENTRIES_IDENT
545#endif
546