Lines Matching +full:addr +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0
6 #include <asm/page-states.h>
63 static bool is_kasan_populate_mode(enum populate_mode mode) in is_kasan_populate_mode() argument
66 return mode >= POPULATE_KASAN_MAP_SHADOW; in is_kasan_populate_mode()
72 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
85 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode) in kasan_populate() argument
90 boot_debug("%-17s 0x%016lx-0x%016lx >> 0x%016lx-0x%016lx\n", get_populate_mode_name(mode), in kasan_populate()
92 pgtable_populate(sha_start, sha_end, mode); in kasan_populate()
136 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, in kasan_pgd_populate_zero_shadow() argument
137 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
139 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pgd_populate_zero_shadow()
140 IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { in kasan_pgd_populate_zero_shadow()
147 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
148 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
150 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_p4d_populate_zero_shadow()
151 IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) { in kasan_p4d_populate_zero_shadow()
158 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, in kasan_pud_populate_zero_shadow() argument
159 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
161 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pud_populate_zero_shadow()
162 IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { in kasan_pud_populate_zero_shadow()
169 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, in kasan_pmd_populate_zero_shadow() argument
170 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
172 if (mode == POPULATE_KASAN_ZERO_SHADOW && in kasan_pmd_populate_zero_shadow()
173 IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { in kasan_pmd_populate_zero_shadow()
180 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
182 if (mode == POPULATE_KASAN_ZERO_SHADOW) { in kasan_pte_populate_zero_shadow()
194 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr, in kasan_pgd_populate_zero_shadow() argument
195 unsigned long end, enum populate_mode mode) in kasan_pgd_populate_zero_shadow() argument
200 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
201 unsigned long end, enum populate_mode mode) in kasan_p4d_populate_zero_shadow() argument
206 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr, in kasan_pud_populate_zero_shadow() argument
207 unsigned long end, enum populate_mode mode) in kasan_pud_populate_zero_shadow() argument
212 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, in kasan_pmd_populate_zero_shadow() argument
213 unsigned long end, enum populate_mode mode) in kasan_pmd_populate_zero_shadow() argument
218 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) in kasan_pte_populate_zero_shadow() argument
266 static unsigned long resolve_pa_may_alloc(unsigned long addr, unsigned long size, in resolve_pa_may_alloc() argument
267 enum populate_mode mode) in resolve_pa_may_alloc() argument
269 switch (mode) { in resolve_pa_may_alloc()
273 return addr; in resolve_pa_may_alloc()
275 return __lowcore_pa(addr); in resolve_pa_may_alloc()
277 return __abs_lowcore_pa(addr); in resolve_pa_may_alloc()
279 return __kernel_pa(addr); in resolve_pa_may_alloc()
281 return __identity_pa(addr); in resolve_pa_may_alloc()
285 addr = physmem_alloc(RR_VMEM, size, size, size == PAGE_SIZE); in resolve_pa_may_alloc()
286 if (addr) { in resolve_pa_may_alloc()
287 memset((void *)addr, 0, size); in resolve_pa_may_alloc()
288 return addr; in resolve_pa_may_alloc()
297 static bool large_page_mapping_allowed(enum populate_mode mode) in large_page_mapping_allowed() argument
299 switch (mode) { in large_page_mapping_allowed()
312 static unsigned long try_get_large_pud_pa(pud_t *pu_dir, unsigned long addr, unsigned long end, in try_get_large_pud_pa() argument
313 enum populate_mode mode) in try_get_large_pud_pa() argument
315 unsigned long pa, size = end - addr; in try_get_large_pud_pa()
317 if (!machine.has_edat2 || !large_page_mapping_allowed(mode) || in try_get_large_pud_pa()
318 !IS_ALIGNED(addr, PUD_SIZE) || (size < PUD_SIZE)) in try_get_large_pud_pa()
321 pa = resolve_pa_may_alloc(addr, size, mode); in try_get_large_pud_pa()
328 static unsigned long try_get_large_pmd_pa(pmd_t *pm_dir, unsigned long addr, unsigned long end, in try_get_large_pmd_pa() argument
329 enum populate_mode mode) in try_get_large_pmd_pa() argument
331 unsigned long pa, size = end - addr; in try_get_large_pmd_pa()
333 if (!machine.has_edat1 || !large_page_mapping_allowed(mode) || in try_get_large_pmd_pa()
334 !IS_ALIGNED(addr, PMD_SIZE) || (size < PMD_SIZE)) in try_get_large_pmd_pa()
337 pa = resolve_pa_may_alloc(addr, size, mode); in try_get_large_pmd_pa()
344 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end, in pgtable_pte_populate() argument
345 enum populate_mode mode) in pgtable_pte_populate() argument
350 pte = pte_offset_kernel(pmd, addr); in pgtable_pte_populate()
351 for (; addr < end; addr += PAGE_SIZE, pte++) { in pgtable_pte_populate()
353 if (kasan_pte_populate_zero_shadow(pte, mode)) in pgtable_pte_populate()
355 entry = __pte(resolve_pa_may_alloc(addr, PAGE_SIZE, mode)); in pgtable_pte_populate()
361 if (mode == POPULATE_IDENTITY) in pgtable_pte_populate()
365 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end, in pgtable_pmd_populate() argument
366 enum populate_mode mode) in pgtable_pmd_populate() argument
372 pmd = pmd_offset(pud, addr); in pgtable_pmd_populate()
373 for (; addr < end; addr = next, pmd++) { in pgtable_pmd_populate()
374 next = pmd_addr_end(addr, end); in pgtable_pmd_populate()
376 if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode)) in pgtable_pmd_populate()
378 pa = try_get_large_pmd_pa(pmd, addr, next, mode); in pgtable_pmd_populate()
391 pgtable_pte_populate(pmd, addr, next, mode); in pgtable_pmd_populate()
393 if (mode == POPULATE_IDENTITY) in pgtable_pmd_populate()
397 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, in pgtable_pud_populate() argument
398 enum populate_mode mode) in pgtable_pud_populate() argument
404 pud = pud_offset(p4d, addr); in pgtable_pud_populate()
405 for (; addr < end; addr = next, pud++) { in pgtable_pud_populate()
406 next = pud_addr_end(addr, end); in pgtable_pud_populate()
408 if (kasan_pud_populate_zero_shadow(pud, addr, next, mode)) in pgtable_pud_populate()
410 pa = try_get_large_pud_pa(pud, addr, next, mode); in pgtable_pud_populate()
423 pgtable_pmd_populate(pud, addr, next, mode); in pgtable_pud_populate()
425 if (mode == POPULATE_IDENTITY) in pgtable_pud_populate()
429 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end, in pgtable_p4d_populate() argument
430 enum populate_mode mode) in pgtable_p4d_populate() argument
436 p4d = p4d_offset(pgd, addr); in pgtable_p4d_populate()
437 for (; addr < end; addr = next, p4d++) { in pgtable_p4d_populate()
438 next = p4d_addr_end(addr, end); in pgtable_p4d_populate()
440 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) in pgtable_p4d_populate()
445 pgtable_pud_populate(p4d, addr, next, mode); in pgtable_p4d_populate()
449 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode) in pgtable_populate() argument
455 if (!is_kasan_populate_mode(mode)) { in pgtable_populate()
456 boot_debug("%-17s 0x%016lx-0x%016lx -> 0x%016lx-0x%016lx\n", in pgtable_populate()
457 get_populate_mode_name(mode), addr, end, in pgtable_populate()
458 resolve_pa_may_alloc(addr, 0, mode), in pgtable_populate()
459 resolve_pa_may_alloc(end - 1, 0, mode) + 1); in pgtable_populate()
462 pgd = pgd_offset(&init_mm, addr); in pgtable_populate()
463 for (; addr < end; addr = next, pgd++) { in pgtable_populate()
464 next = pgd_addr_end(addr, end); in pgtable_populate()
466 if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode)) in pgtable_populate()
472 if (mode == POPULATE_KASAN_SHALLOW) in pgtable_populate()
475 pgtable_p4d_populate(pgd, addr, next, mode); in pgtable_populate()
489 * Mark whole memory as no-dat. This must be done before any in setup_vmem()
494 __arch_set_page_nodat((void *)start, (end - start) >> PAGE_SHIFT); in setup_vmem()
497 * init_mm->pgd contains virtual address of swapper_pg_dir. in setup_vmem()
555 get_lowcore()->kernel_asce.val = swapper_pg_dir | asce_bits; in setup_vmem()
556 get_lowcore()->user_asce = s390_invalid_asce; in setup_vmem()
558 local_ctl_load(1, &get_lowcore()->kernel_asce); in setup_vmem()
559 local_ctl_load(7, &get_lowcore()->user_asce); in setup_vmem()
560 local_ctl_load(13, &get_lowcore()->kernel_asce); in setup_vmem()
562 init_mm.context.asce = get_lowcore()->kernel_asce.val; in setup_vmem()