Lines Matching full:vm

23 static uint64_t page_align(struct kvm_vm *vm, uint64_t v)  in page_align()  argument
25 return (v + vm->page_size) & ~(vm->page_size - 1); in page_align()
28 static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva) in pgd_index() argument
30 unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift; in pgd_index()
31 uint64_t mask = (1UL << (vm->va_bits - shift)) - 1; in pgd_index()
36 static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva) in pud_index() argument
38 unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift; in pud_index()
39 uint64_t mask = (1UL << (vm->page_shift - 3)) - 1; in pud_index()
41 TEST_ASSERT(vm->pgtable_levels == 4, in pud_index()
42 "Mode %d does not have 4 page table levels", vm->mode); in pud_index()
47 static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva) in pmd_index() argument
49 unsigned int shift = (vm->page_shift - 3) + vm->page_shift; in pmd_index()
50 uint64_t mask = (1UL << (vm->page_shift - 3)) - 1; in pmd_index()
52 TEST_ASSERT(vm->pgtable_levels >= 3, in pmd_index()
53 "Mode %d does not have >= 3 page table levels", vm->mode); in pmd_index()
58 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva) in pte_index() argument
60 uint64_t mask = (1UL << (vm->page_shift - 3)) - 1; in pte_index()
61 return (gva >> vm->page_shift) & mask; in pte_index()
64 static inline bool use_lpa2_pte_format(struct kvm_vm *vm) in use_lpa2_pte_format() argument
66 return (vm->page_size == SZ_4K || vm->page_size == SZ_16K) && in use_lpa2_pte_format()
67 (vm->pa_bits > 48 || vm->va_bits > 48); in use_lpa2_pte_format()
70 static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs) in addr_pte() argument
74 if (use_lpa2_pte_format(vm)) { in addr_pte()
75 pte = pa & GENMASK(49, vm->page_shift); in addr_pte()
79 pte = pa & GENMASK(47, vm->page_shift); in addr_pte()
80 if (vm->page_shift == 16) in addr_pte()
88 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte) in pte_addr() argument
92 if (use_lpa2_pte_format(vm)) { in pte_addr()
93 pa = pte & GENMASK(49, vm->page_shift); in pte_addr()
96 pa = pte & GENMASK(47, vm->page_shift); in pte_addr()
97 if (vm->page_shift == 16) in pte_addr()
104 static uint64_t ptrs_per_pgd(struct kvm_vm *vm) in ptrs_per_pgd() argument
106 unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift; in ptrs_per_pgd()
107 return 1 << (vm->va_bits - shift); in ptrs_per_pgd()
110 static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm) in ptrs_per_pte() argument
112 return 1 << (vm->page_shift - 3); in ptrs_per_pte()
115 void virt_arch_pgd_alloc(struct kvm_vm *vm) in virt_arch_pgd_alloc() argument
117 size_t nr_pages = page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size; in virt_arch_pgd_alloc()
119 if (vm->pgd_created) in virt_arch_pgd_alloc()
122 vm->pgd = vm_phy_pages_alloc(vm, nr_pages, in virt_arch_pgd_alloc()
124 vm->memslots[MEM_REGION_PT]); in virt_arch_pgd_alloc()
125 vm->pgd_created = true; in virt_arch_pgd_alloc()
128 static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, in _virt_pg_map() argument
134 TEST_ASSERT((vaddr % vm->page_size) == 0, in _virt_pg_map()
136 " vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size); in _virt_pg_map()
137 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, in _virt_pg_map()
138 (vaddr >> vm->page_shift)), in _virt_pg_map()
140 TEST_ASSERT((paddr % vm->page_size) == 0, in _virt_pg_map()
142 " paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size); in _virt_pg_map()
143 TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, in _virt_pg_map()
145 " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", in _virt_pg_map()
146 paddr, vm->max_gfn, vm->page_size); in _virt_pg_map()
148 ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8; in _virt_pg_map()
150 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map()
152 switch (vm->pgtable_levels) { in _virt_pg_map()
154 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8; in _virt_pg_map()
156 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map()
159 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8; in _virt_pg_map()
161 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map()
164 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8; in _virt_pg_map()
170 *ptep = addr_pte(vm, paddr, (attr_idx << 2) | (1 << 10) | 3); /* AF */ in _virt_pg_map()
173 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) in virt_arch_pg_map() argument
177 _virt_pg_map(vm, vaddr, paddr, attr_idx); in virt_arch_pg_map()
180 uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva) in virt_get_pte_hva() argument
184 if (!vm->pgd_created) in virt_get_pte_hva()
187 ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8; in virt_get_pte_hva()
191 switch (vm->pgtable_levels) { in virt_get_pte_hva()
193 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8; in virt_get_pte_hva()
198 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8; in virt_get_pte_hva()
203 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8; in virt_get_pte_hva()
214 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); in virt_get_pte_hva()
218 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) in addr_arch_gva2gpa() argument
220 uint64_t *ptep = virt_get_pte_hva(vm, gva); in addr_arch_gva2gpa()
222 return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1)); in addr_arch_gva2gpa()
225 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level) in pte_dump() argument
234 for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) { in pte_dump()
235 ptep = addr_gpa2hva(vm, pte); in pte_dump()
239 pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1); in pte_dump()
244 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) in virt_arch_dump() argument
246 int level = 4 - (vm->pgtable_levels - 1); in virt_arch_dump()
249 if (!vm->pgd_created) in virt_arch_dump()
252 for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pgd(vm) * 8; pgd += 8) { in virt_arch_dump()
253 ptep = addr_gpa2hva(vm, pgd); in virt_arch_dump()
257 pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level); in virt_arch_dump()
264 struct kvm_vm *vm = vcpu->vm; in aarch64_vcpu_setup() local
272 vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred); in aarch64_vcpu_setup()
288 switch (vm->mode) { in aarch64_vcpu_setup()
312 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); in aarch64_vcpu_setup()
315 ttbr0_el1 = vm->pgd & GENMASK(47, vm->page_shift); in aarch64_vcpu_setup()
318 switch (vm->mode) { in aarch64_vcpu_setup()
323 ttbr0_el1 |= FIELD_GET(GENMASK(51, 48), vm->pgd) << 2; in aarch64_vcpu_setup()
342 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); in aarch64_vcpu_setup()
348 tcr_el1 |= (64 - vm->va_bits) /* T0SZ */; in aarch64_vcpu_setup()
349 if (use_lpa2_pte_format(vm)) in aarch64_vcpu_setup()
375 static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, in __aarch64_vcpu_add() argument
380 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id); in __aarch64_vcpu_add()
382 stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size : in __aarch64_vcpu_add()
383 vm->page_size; in __aarch64_vcpu_add()
384 stack_vaddr = __vm_vaddr_alloc(vm, stack_size, in __aarch64_vcpu_add()
394 struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, in aarch64_vcpu_add() argument
397 struct kvm_vcpu *vcpu = __aarch64_vcpu_add(vm, vcpu_id, init); in aarch64_vcpu_add()
404 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id) in vm_arch_vcpu_add() argument
406 return __aarch64_vcpu_add(vm, vcpu_id, NULL); in vm_arch_vcpu_add()
496 void vm_init_descriptor_tables(struct kvm_vm *vm) in vm_init_descriptor_tables() argument
498 vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers), in vm_init_descriptor_tables()
499 vm->page_size, MEM_REGION_DATA); in vm_init_descriptor_tables()
501 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; in vm_init_descriptor_tables()
504 void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec, in vm_install_sync_handler() argument
507 struct handlers *handlers = addr_gva2hva(vm, vm->handlers); in vm_install_sync_handler()
515 void vm_install_exception_handler(struct kvm_vm *vm, int vector, in vm_install_exception_handler() argument
518 struct handlers *handlers = addr_gva2hva(vm, vm->handlers); in vm_install_exception_handler()
633 void vm_vaddr_populate_bitmap(struct kvm_vm *vm) in vm_vaddr_populate_bitmap() argument
639 sparsebit_set_num(vm->vpages_valid, 0, in vm_vaddr_populate_bitmap()
640 (1ULL << vm->va_bits) >> vm->page_shift); in vm_vaddr_populate_bitmap()