Lines Matching +full:next +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0
6 #include <asm/page-states.h>
63 static bool is_kasan_populate_mode(enum populate_mode mode) in is_kasan_populate_mode() argument
66 return mode >= POPULATE_KASAN_MAP_SHADOW; in is_kasan_populate_mode()
72 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
85 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode) in kasan_populate() argument
90 boot_debug("%-17s 0x%016lx-0x%016lx >> 0x%016lx-0x%016lx\n", get_populate_mode_name(mode), in kasan_populate()
92 pgtable_populate(sha_start, sha_end, mode); in kasan_populate()
137 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
139 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pgd_populate_zero_shadow()
140 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { in kasan_pgd_populate_zero_shadow()
148 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
150 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_p4d_populate_zero_shadow()
151 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { in kasan_p4d_populate_zero_shadow()
159 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
161 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pud_populate_zero_shadow()
162 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { in kasan_pud_populate_zero_shadow()
170 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
172 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pmd_populate_zero_shadow()
173 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { in kasan_pmd_populate_zero_shadow()
180 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
182 if (mode == POPULATE_KASAN_ZERO_SHADOW) { in kasan_pte_populate_zero_shadow()
195 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
201 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
207 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
213 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
218 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
267 enum populate_mode mode) in resolve_pa_may_alloc() argument
269 switch (mode) { in resolve_pa_may_alloc()
297 static bool large_page_mapping_allowed(enum populate_mode mode) in large_page_mapping_allowed() argument
299 switch (mode) { in large_page_mapping_allowed()
313 enum populate_mode mode) in try_get_large_pud_pa() argument
315 unsigned long pa, size = end - addr; in try_get_large_pud_pa()
317 if (!machine.has_edat2 || !large_page_mapping_allowed(mode) || in try_get_large_pud_pa()
321 pa = resolve_pa_may_alloc(addr, size, mode); in try_get_large_pud_pa()
329 enum populate_mode mode) in try_get_large_pmd_pa() argument
331 unsigned long pa, size = end - addr; in try_get_large_pmd_pa()
333 if (!machine.has_edat1 || !large_page_mapping_allowed(mode) || in try_get_large_pmd_pa()
337 pa = resolve_pa_may_alloc(addr, size, mode); in try_get_large_pmd_pa()
345 enum populate_mode mode) in pgtable_pte_populate() argument
353 if (kasan_pte_populate_zero_shadow(pte, mode)) in pgtable_pte_populate()
355 entry = __pte(resolve_pa_may_alloc(addr, PAGE_SIZE, mode)); in pgtable_pte_populate()
361 if (mode == POPULATE_IDENTITY) in pgtable_pte_populate()
366 enum populate_mode mode) in pgtable_pmd_populate() argument
368 unsigned long pa, next, pages = 0; in pgtable_pmd_populate() local
373 for (; addr < end; addr = next, pmd++) { in pgtable_pmd_populate()
374 next = pmd_addr_end(addr, end); in pgtable_pmd_populate()
376 if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) in pgtable_pmd_populate()
378 pa = try_get_large_pmd_pa(pmd, addr, next, mode); in pgtable_pmd_populate()
391 pgtable_pte_populate(pmd, addr, next, mode); in pgtable_pmd_populate()
393 if (mode == POPULATE_IDENTITY) in pgtable_pmd_populate()
398 enum populate_mode mode) in pgtable_pud_populate() argument
400 unsigned long pa, next, pages = 0; in pgtable_pud_populate() local
405 for (; addr < end; addr = next, pud++) { in pgtable_pud_populate()
406 next = pud_addr_end(addr, end); in pgtable_pud_populate()
408 if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) in pgtable_pud_populate()
410 pa = try_get_large_pud_pa(pud, addr, next, mode); in pgtable_pud_populate()
423 pgtable_pmd_populate(pud, addr, next, mode); in pgtable_pud_populate()
425 if (mode == POPULATE_IDENTITY) in pgtable_pud_populate()
430 enum populate_mode mode) in pgtable_p4d_populate() argument
432 unsigned long next; in pgtable_p4d_populate() local
437 for (; addr < end; addr = next, p4d++) { in pgtable_p4d_populate()
438 next = p4d_addr_end(addr, end); in pgtable_p4d_populate()
440 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) in pgtable_p4d_populate()
445 pgtable_pud_populate(p4d, addr, next, mode); in pgtable_p4d_populate()
449 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) in pgtable_populate() argument
451 unsigned long next; in pgtable_populate() local
455 if (!is_kasan_populate_mode(mode)) { in pgtable_populate()
456 boot_debug("%-17s 0x%016lx-0x%016lx -> 0x%016lx-0x%016lx\n", in pgtable_populate()
457 get_populate_mode_name(mode), addr, end, in pgtable_populate()
458 resolve_pa_may_alloc(addr, 0, mode), in pgtable_populate()
459 resolve_pa_may_alloc(end - 1, 0, mode) + 1); in pgtable_populate()
463 for (; addr < end; addr = next, pgd++) { in pgtable_populate()
464 next = pgd_addr_end(addr, end); in pgtable_populate()
466 if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) in pgtable_populate()
472 if (mode == POPULATE_KASAN_SHALLOW) in pgtable_populate()
475 pgtable_p4d_populate(pgd, addr, next, mode); in pgtable_populate()
489 * Mark whole memory as no-dat. This must be done before any in setup_vmem()
494 __arch_set_page_nodat((void *)start, (end - start) >> PAGE_SHIFT); in setup_vmem()
497 * init_mm->pgd contains virtual address of swapper_pg_dir. in setup_vmem()
555 get_lowcore()->kernel_asce.val = swapper_pg_dir | asce_bits; in setup_vmem()
556 get_lowcore()->user_asce = s390_invalid_asce; in setup_vmem()
558 local_ctl_load(1, &get_lowcore()->kernel_asce); in setup_vmem()
559 local_ctl_load(7, &get_lowcore()->user_asce); in setup_vmem()
560 local_ctl_load(13, &get_lowcore()->kernel_asce); in setup_vmem()
562 init_mm.context.asce = get_lowcore()->kernel_asce.val; in setup_vmem()