1 // SPDX-License-Identifier: GPL-2.0-only
6 #include <asm/msr-index.h>
64  *   vm - The VM to allocate guest-virtual addresses in.
67  *   p_vmx_gva - The guest virtual address for the struct vmx_pages.
73 vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva)  in vcpu_alloc_vmx()  argument
75 	vm_vaddr_t vmx_gva = vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
76 	struct vmx_pages *vmx = addr_gva2hva(vm, vmx_gva);  in vcpu_alloc_vmx()
79 	vmx->vmxon = (void *)vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
80 	vmx->vmxon_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmxon);  in vcpu_alloc_vmx()
81 	vmx->vmxon_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmxon);  in vcpu_alloc_vmx()
84 	vmx->vmcs = (void *)vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
85 	vmx->vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmcs);  in vcpu_alloc_vmx()
86 	vmx->vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmcs);  in vcpu_alloc_vmx()
89 	vmx->msr = (void *)vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
90 	vmx->msr_hva = addr_gva2hva(vm, (uintptr_t)vmx->msr);  in vcpu_alloc_vmx()
91 	vmx->msr_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->msr);  in vcpu_alloc_vmx()
92 	memset(vmx->msr_hva, 0, getpagesize());  in vcpu_alloc_vmx()
95 	vmx->shadow_vmcs = (void *)vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
96 	vmx->shadow_vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->shadow_vmcs);  in vcpu_alloc_vmx()
97 	vmx->shadow_vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->shadow_vmcs);  in vcpu_alloc_vmx()
100 	vmx->vmread = (void *)vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
101 	vmx->vmread_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmread);  in vcpu_alloc_vmx()
102 	vmx->vmread_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmread);  in vcpu_alloc_vmx()
103 	memset(vmx->vmread_hva, 0, getpagesize());  in vcpu_alloc_vmx()
105 	vmx->vmwrite = (void *)vm_vaddr_alloc_page(vm);  in vcpu_alloc_vmx()
106 	vmx->vmwrite_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmwrite);  in vcpu_alloc_vmx()
107 	vmx->vmwrite_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmwrite);  in vcpu_alloc_vmx()
108 	memset(vmx->vmwrite_hva, 0, getpagesize());  in vcpu_alloc_vmx()
123 	 * - Bit X is 1 in _FIXED0: bit X is fixed to 1 in CRx.  in prepare_for_vmx_operation()
124 	 * - Bit X is 0 in _FIXED1: bit X is fixed to 0 in CRx.  in prepare_for_vmx_operation()
151 	*(uint32_t *)(vmx->vmxon) = vmcs_revision();  in prepare_for_vmx_operation()
152 	if (vmxon(vmx->vmxon_gpa))  in prepare_for_vmx_operation()
161 	*(uint32_t *)(vmx->vmcs) = vmcs_revision();  in load_vmcs()
162 	if (vmclear(vmx->vmcs_gpa))  in load_vmcs()
165 	if (vmptrld(vmx->vmcs_gpa))  in load_vmcs()
169 	*(uint32_t *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul;  in load_vmcs()
170 	if (vmclear(vmx->shadow_vmcs_gpa))  in load_vmcs()
198 	if (vmx->eptp_gpa) {  in init_vmcs_control_fields()
204 			.address = vmx->eptp_gpa >> PAGE_SHIFT_4K,  in init_vmcs_control_fields()
222 	vmwrite(PAGE_FAULT_ERROR_CODE_MATCH, -1); /* Never match */  in init_vmcs_control_fields()
225 		VM_EXIT_HOST_ADDR_SPACE_SIZE);	  /* 64-bit host */  in init_vmcs_control_fields()
229 		VM_ENTRY_IA32E_MODE);		  /* 64-bit guest */  in init_vmcs_control_fields()
239 	vmwrite(MSR_BITMAP, vmx->msr_gpa);  in init_vmcs_control_fields()
240 	vmwrite(VMREAD_BITMAP, vmx->vmread_gpa);  in init_vmcs_control_fields()
241 	vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa);  in init_vmcs_control_fields()
302 	vmwrite(VMCS_LINK_POINTER, -1ll);  in init_vmcs_guest_state()
309 	vmwrite(GUEST_ES_LIMIT, -1);  in init_vmcs_guest_state()
310 	vmwrite(GUEST_CS_LIMIT, -1);  in init_vmcs_guest_state()
311 	vmwrite(GUEST_SS_LIMIT, -1);  in init_vmcs_guest_state()
312 	vmwrite(GUEST_DS_LIMIT, -1);  in init_vmcs_guest_state()
313 	vmwrite(GUEST_FS_LIMIT, -1);  in init_vmcs_guest_state()
314 	vmwrite(GUEST_GS_LIMIT, -1);  in init_vmcs_guest_state()
315 	vmwrite(GUEST_LDTR_LIMIT, -1);  in init_vmcs_guest_state()
365 static void nested_create_pte(struct kvm_vm *vm,  in nested_create_pte()  argument
372 	if (!pte->readable) {  in nested_create_pte()
373 		pte->writable = true;  in nested_create_pte()
374 		pte->readable = true;  in nested_create_pte()
375 		pte->executable = true;  in nested_create_pte()
376 		pte->page_size = (current_level == target_level);  in nested_create_pte()
377 		if (pte->page_size)  in nested_create_pte()
378 			pte->address = paddr >> vm->page_shift;  in nested_create_pte()
380 			pte->address = vm_alloc_page_table(vm) >> vm->page_shift;  in nested_create_pte()
390 		TEST_ASSERT(!pte->page_size,  in nested_create_pte()
397 void __nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm,  in __nested_pg_map()  argument
401 	struct eptPageTableEntry *pt = vmx->eptp_hva, *pte;  in __nested_pg_map()
404 	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "  in __nested_pg_map()
405 		    "unknown or unsupported guest mode, mode: 0x%x", vm->mode);  in __nested_pg_map()
408 		    "Nested physical address 0x%lx requires 5-level paging",  in __nested_pg_map()
414 	TEST_ASSERT((nested_paddr >> vm->page_shift) <= vm->max_gfn,  in __nested_pg_map()
416 		    "  nested_paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",  in __nested_pg_map()
417 		    paddr, vm->max_gfn, vm->page_size);  in __nested_pg_map()
422 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,  in __nested_pg_map()
424 		    "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",  in __nested_pg_map()
425 		    paddr, vm->max_gfn, vm->page_size);  in __nested_pg_map()
427 	for (int level = PG_LEVEL_512G; level >= PG_LEVEL_4K; level--) {  in __nested_pg_map()
431 		nested_create_pte(vm, pte, nested_paddr, paddr, level, target_level);  in __nested_pg_map()
433 		if (pte->page_size)  in __nested_pg_map()
436 		pt = addr_gpa2hva(vm, pte->address * vm->page_size);  in __nested_pg_map()
443 	pte->accessed = true;  in __nested_pg_map()
444 	pte->dirty = true;  in __nested_pg_map()
448 void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm,  in nested_pg_map()  argument
451 	__nested_pg_map(vmx, vm, nested_paddr, paddr, PG_LEVEL_4K);  in nested_pg_map()
455  * Map a range of EPT guest physical addresses to the VM's physical address
458  *   vm - Virtual Machine
459  *   nested_paddr - Nested guest physical address to map
460  *   paddr - VM Physical Address
461  *   size - The size of the range to map
462  *   level - The level at which to map the range
468  * Within the VM given by vm, creates a nested guest translation for the
471 void __nested_map(struct vmx_pages *vmx, struct kvm_vm *vm,  in __nested_map()  argument
481 	while (npages--) {  in __nested_map()
482 		__nested_pg_map(vmx, vm, nested_paddr, paddr, level);  in __nested_map()
488 void nested_map(struct vmx_pages *vmx, struct kvm_vm *vm,  in nested_map()  argument
491 	__nested_map(vmx, vm, nested_paddr, paddr, size, PG_LEVEL_4K);  in nested_map()
495  * physical pages in VM.
497 void nested_map_memslot(struct vmx_pages *vmx, struct kvm_vm *vm,  in nested_map_memslot()  argument
502 		memslot2region(vm, memslot);  in nested_map_memslot()
504 	i = (region->region.guest_phys_addr >> vm->page_shift) - 1;  in nested_map_memslot()
505 	last = i + (region->region.memory_size >> vm->page_shift);  in nested_map_memslot()
507 		i = sparsebit_next_clear(region->unused_phy_pages, i);  in nested_map_memslot()
511 		nested_map(vmx, vm,  in nested_map_memslot()
512 			   (uint64_t)i << vm->page_shift,  in nested_map_memslot()
513 			   (uint64_t)i << vm->page_shift,  in nested_map_memslot()
514 			   1 << vm->page_shift);  in nested_map_memslot()
518 /* Identity map a region with 1GiB Pages. */
519 void nested_identity_map_1g(struct vmx_pages *vmx, struct kvm_vm *vm,  in nested_identity_map_1g()  argument
522 	__nested_map(vmx, vm, addr, addr, size, PG_LEVEL_1G);  in nested_identity_map_1g()
537 void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm,  in prepare_eptp()  argument
542 	vmx->eptp = (void *)vm_vaddr_alloc_page(vm);  in prepare_eptp()
543 	vmx->eptp_hva = addr_gva2hva(vm, (uintptr_t)vmx->eptp);  in prepare_eptp()
544 	vmx->eptp_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->eptp);  in prepare_eptp()
547 void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm)  in prepare_virtualize_apic_accesses()  argument
549 	vmx->apic_access = (void *)vm_vaddr_alloc_page(vm);  in prepare_virtualize_apic_accesses()
550 	vmx->apic_access_hva = addr_gva2hva(vm, (uintptr_t)vmx->apic_access);  in prepare_virtualize_apic_accesses()
551 	vmx->apic_access_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->apic_access);  in prepare_virtualize_apic_accesses()