109 	((struct kvm_vcpu *)(ctxt)->vcpu)
129 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
130 static void process_nmi(struct kvm_vcpu *vcpu);
131 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
132 static void store_regs(struct kvm_vcpu *vcpu);
133 static int sync_regs(struct kvm_vcpu *vcpu);
134 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
136 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
137 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
258 	STATS_DESC_COUNTER(VCPU, pf_taken),
259 	STATS_DESC_COUNTER(VCPU, pf_fixed),
260 	STATS_DESC_COUNTER(VCPU, pf_emulate),
261 	STATS_DESC_COUNTER(VCPU, pf_spurious),
262 	STATS_DESC_COUNTER(VCPU, pf_fast),
263 	STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
264 	STATS_DESC_COUNTER(VCPU, pf_guest),
265 	STATS_DESC_COUNTER(VCPU, tlb_flush),
266 	STATS_DESC_COUNTER(VCPU, invlpg),
267 	STATS_DESC_COUNTER(VCPU, exits),
268 	STATS_DESC_COUNTER(VCPU, io_exits),
269 	STATS_DESC_COUNTER(VCPU, mmio_exits),
270 	STATS_DESC_COUNTER(VCPU, signal_exits),
271 	STATS_DESC_COUNTER(VCPU, irq_window_exits),
272 	STATS_DESC_COUNTER(VCPU, nmi_window_exits),
273 	STATS_DESC_COUNTER(VCPU, l1d_flush),
274 	STATS_DESC_COUNTER(VCPU, halt_exits),
275 	STATS_DESC_COUNTER(VCPU, request_irq_exits),
276 	STATS_DESC_COUNTER(VCPU, irq_exits),
277 	STATS_DESC_COUNTER(VCPU, host_state_reload),
278 	STATS_DESC_COUNTER(VCPU, fpu_reload),
279 	STATS_DESC_COUNTER(VCPU, insn_emulation),
280 	STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
281 	STATS_DESC_COUNTER(VCPU, hypercalls),
282 	STATS_DESC_COUNTER(VCPU, irq_injections),
283 	STATS_DESC_COUNTER(VCPU, nmi_injections),
284 	STATS_DESC_COUNTER(VCPU, req_event),
285 	STATS_DESC_COUNTER(VCPU, nested_run),
286 	STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
287 	STATS_DESC_COUNTER(VCPU, directed_yield_successful),
288 	STATS_DESC_COUNTER(VCPU, preemption_reported),
289 	STATS_DESC_COUNTER(VCPU, preemption_other),
290 	STATS_DESC_IBOOLEAN(VCPU, guest_mode),
291 	STATS_DESC_COUNTER(VCPU, notify_window_exits),
461  * patch, are immutable once the vCPU model is defined.
495 typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data,
498 static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr,  in kvm_do_msr_access()  argument
512 	ret = msr_access_fn(vcpu, msr, data, host_initiated);  in kvm_do_msr_access()
553 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)  in kvm_async_pf_hash_reset()  argument
557 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_async_pf_hash_reset()
736 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,  in kvm_deliver_exception_payload()  argument
749 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;  in kvm_deliver_exception_payload()
766 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
767 		vcpu->arch.dr6 |= ex->payload;  in kvm_deliver_exception_payload()
768 		vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;  in kvm_deliver_exception_payload()
776 		vcpu->arch.dr6 &= ~BIT(12);  in kvm_deliver_exception_payload()
779 		vcpu->arch.cr2 = ex->payload;  in kvm_deliver_exception_payload()
788 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,  in kvm_queue_exception_vmexit()  argument
792 	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;  in kvm_queue_exception_vmexit()
803 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,  in kvm_multiple_exception()  argument
810 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_multiple_exception()
819 	if (!reinject && is_guest_mode(vcpu) &&  in kvm_multiple_exception()
820 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {  in kvm_multiple_exception()
821 		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,  in kvm_multiple_exception()
826 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {  in kvm_multiple_exception()
836 			WARN_ON_ONCE(kvm_is_exception_pending(vcpu));  in kvm_multiple_exception()
837 			vcpu->arch.exception.injected = true;  in kvm_multiple_exception()
847 			vcpu->arch.exception.pending = true;  in kvm_multiple_exception()
848 			vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
850 		vcpu->arch.exception.has_error_code = has_error;  in kvm_multiple_exception()
851 		vcpu->arch.exception.vector = nr;  in kvm_multiple_exception()
852 		vcpu->arch.exception.error_code = error_code;  in kvm_multiple_exception()
853 		vcpu->arch.exception.has_payload = has_payload;  in kvm_multiple_exception()
854 		vcpu->arch.exception.payload = payload;  in kvm_multiple_exception()
855 		if (!is_guest_mode(vcpu))  in kvm_multiple_exception()
856 			kvm_deliver_exception_payload(vcpu,  in kvm_multiple_exception()
857 						      &vcpu->arch.exception);  in kvm_multiple_exception()
862 	prev_nr = vcpu->arch.exception.vector;  in kvm_multiple_exception()
865 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_multiple_exception()
876 		vcpu->arch.exception.injected = false;  in kvm_multiple_exception()
877 		vcpu->arch.exception.pending = false;  in kvm_multiple_exception()
879 		kvm_queue_exception_e(vcpu, DF_VECTOR, 0);  in kvm_multiple_exception()
888 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_queue_exception()  argument
890 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);  in kvm_queue_exception()
894 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)  in kvm_requeue_exception()  argument
896 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);  in kvm_requeue_exception()
900 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_p()  argument
903 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);  in kvm_queue_exception_p()
907 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,  in kvm_queue_exception_e_p()  argument
910 	kvm_multiple_exception(vcpu, nr, true, error_code,  in kvm_queue_exception_e_p()
914 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)  in kvm_complete_insn_gp()  argument
917 		kvm_inject_gp(vcpu, 0);  in kvm_complete_insn_gp()
919 		return kvm_skip_emulated_instruction(vcpu);  in kvm_complete_insn_gp()
925 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)  in complete_emulated_insn_gp()  argument
928 		kvm_inject_gp(vcpu, 0);  in complete_emulated_insn_gp()
932 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |  in complete_emulated_insn_gp()
936 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)  in kvm_inject_page_fault()  argument
938 	++vcpu->stat.pf_guest;  in kvm_inject_page_fault()
944 	if (is_guest_mode(vcpu) && fault->async_page_fault)  in kvm_inject_page_fault()
945 		kvm_queue_exception_vmexit(vcpu, PF_VECTOR,  in kvm_inject_page_fault()
949 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,  in kvm_inject_page_fault()
953 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,  in kvm_inject_emulated_page_fault()  argument
959 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :  in kvm_inject_emulated_page_fault()
960 					       vcpu->arch.walk_mmu;  in kvm_inject_emulated_page_fault()
968 		kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,  in kvm_inject_emulated_page_fault()
971 	fault_mmu->inject_page_fault(vcpu, fault);  in kvm_inject_emulated_page_fault()
975 void kvm_inject_nmi(struct kvm_vcpu *vcpu)  in kvm_inject_nmi()  argument
977 	atomic_inc(&vcpu->arch.nmi_queued);  in kvm_inject_nmi()
978 	kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_inject_nmi()
981 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_queue_exception_e()  argument
983 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);  in kvm_queue_exception_e()
987 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)  in kvm_requeue_exception_e()  argument
989 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);  in kvm_requeue_exception_e()
997 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)  in kvm_require_cpl()  argument
999 	if (kvm_x86_call(get_cpl)(vcpu) <= required_cpl)  in kvm_require_cpl()
1001 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in kvm_require_cpl()
1005 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)  in kvm_require_dr()  argument
1007 	if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))  in kvm_require_dr()
1010 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_require_dr()
1015 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)  in pdptr_rsvd_bits()  argument
1017 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);  in pdptr_rsvd_bits()
1023 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)  in load_pdptrs()  argument
1025 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in load_pdptrs()
1036 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),  in load_pdptrs()
1042 	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,  in load_pdptrs()
1049 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {  in load_pdptrs()
1059 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);  in load_pdptrs()
1062 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in load_pdptrs()
1063 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);  in load_pdptrs()
1064 	vcpu->arch.pdptrs_from_userspace = false;  in load_pdptrs()
1070 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_is_valid_cr0()  argument
1083 	return kvm_x86_call(is_valid_cr0)(vcpu, cr0);  in kvm_is_valid_cr0()
1086 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)  in kvm_post_set_cr0()  argument
1101 			kvm_init_mmu(vcpu);  in kvm_post_set_cr0()
1107 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_post_set_cr0()
1108 		kvm_async_pf_hash_reset(vcpu);  in kvm_post_set_cr0()
1115 			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr0()
1119 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr0()
1123 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)  in kvm_set_cr0()  argument
1125 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_set_cr0()
1127 	if (!kvm_is_valid_cr0(vcpu, cr0))  in kvm_set_cr0()
1136 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&  in kvm_set_cr0()
1140 		if (!is_pae(vcpu))  in kvm_set_cr0()
1142 		kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in kvm_set_cr0()
1147 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&  in kvm_set_cr0()
1148 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&  in kvm_set_cr0()
1149 	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr0()
1153 	    (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))  in kvm_set_cr0()
1156 	kvm_x86_call(set_cr0)(vcpu, cr0);  in kvm_set_cr0()
1158 	kvm_post_set_cr0(vcpu, old_cr0, cr0);  in kvm_set_cr0()
1164 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)  in kvm_lmsw()  argument
1166 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));  in kvm_lmsw()
1170 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_guest_xsave_state()  argument
1172 	if (vcpu->arch.guest_state_protected)  in kvm_load_guest_xsave_state()
1175 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_guest_xsave_state()
1177 		if (vcpu->arch.xcr0 != kvm_host.xcr0)  in kvm_load_guest_xsave_state()
1178 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);  in kvm_load_guest_xsave_state()
1180 		if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_guest_xsave_state()
1181 		    vcpu->arch.ia32_xss != kvm_host.xss)  in kvm_load_guest_xsave_state()
1182 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);  in kvm_load_guest_xsave_state()
1186 	    vcpu->arch.pkru != vcpu->arch.host_pkru &&  in kvm_load_guest_xsave_state()
1187 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_guest_xsave_state()
1188 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))  in kvm_load_guest_xsave_state()
1189 		wrpkru(vcpu->arch.pkru);  in kvm_load_guest_xsave_state()
1193 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)  in kvm_load_host_xsave_state()  argument
1195 	if (vcpu->arch.guest_state_protected)  in kvm_load_host_xsave_state()
1199 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||  in kvm_load_host_xsave_state()
1200 	     kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {  in kvm_load_host_xsave_state()
1201 		vcpu->arch.pkru = rdpkru();  in kvm_load_host_xsave_state()
1202 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)  in kvm_load_host_xsave_state()
1203 			wrpkru(vcpu->arch.host_pkru);  in kvm_load_host_xsave_state()
1206 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {  in kvm_load_host_xsave_state()
1208 		if (vcpu->arch.xcr0 != kvm_host.xcr0)  in kvm_load_host_xsave_state()
1211 		if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) &&  in kvm_load_host_xsave_state()
1212 		    vcpu->arch.ia32_xss != kvm_host.xss)  in kvm_load_host_xsave_state()
1220 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)  in kvm_guest_supported_xfd()  argument
1222 	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;  in kvm_guest_supported_xfd()
1226 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)  in __kvm_set_xcr()  argument
1229 	u64 old_xcr0 = vcpu->arch.xcr0;  in __kvm_set_xcr()
1245 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;  in __kvm_set_xcr()
1264 	vcpu->arch.xcr0 = xcr0;  in __kvm_set_xcr()
1267 		kvm_update_cpuid_runtime(vcpu);  in __kvm_set_xcr()
1271 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)  in kvm_emulate_xsetbv()  argument
1274 	if (kvm_x86_call(get_cpl)(vcpu) != 0 ||  in kvm_emulate_xsetbv()
1275 	    __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {  in kvm_emulate_xsetbv()
1276 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_xsetbv()
1280 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_xsetbv()
1284 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_is_valid_cr4()  argument
1286 	return __kvm_is_valid_cr4(vcpu, cr4) &&  in kvm_is_valid_cr4()
1287 	       kvm_x86_call(is_valid_cr4)(vcpu, cr4);  in kvm_is_valid_cr4()
1290 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)  in kvm_post_set_cr4()  argument
1293 		kvm_mmu_reset_context(vcpu);  in kvm_post_set_cr4()
1305 		kvm_mmu_unload(vcpu);  in kvm_post_set_cr4()
1317 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_post_set_cr4()
1327 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_post_set_cr4()
1332 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)  in kvm_set_cr4()  argument
1334 	unsigned long old_cr4 = kvm_read_cr4(vcpu);  in kvm_set_cr4()
1336 	if (!kvm_is_valid_cr4(vcpu, cr4))  in kvm_set_cr4()
1339 	if (is_long_mode(vcpu)) {  in kvm_set_cr4()
1344 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)  in kvm_set_cr4()
1346 		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))  in kvm_set_cr4()
1351 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))  in kvm_set_cr4()
1355 	kvm_x86_call(set_cr4)(vcpu, cr4);  in kvm_set_cr4()
1357 	kvm_post_set_cr4(vcpu, old_cr4, cr4);  in kvm_set_cr4()
1363 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)  in kvm_invalidate_pcid()  argument
1365 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_invalidate_pcid()
1377 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_invalidate_pcid()
1386 	if (kvm_get_active_pcid(vcpu) == pcid) {  in kvm_invalidate_pcid()
1387 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);  in kvm_invalidate_pcid()
1388 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_invalidate_pcid()
1396 	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))  in kvm_invalidate_pcid()
1400 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)  in kvm_invalidate_pcid()
1403 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);  in kvm_invalidate_pcid()
1406 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)  in kvm_set_cr3()  argument
1411 	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {  in kvm_set_cr3()
1419 	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))  in kvm_set_cr3()
1425 	 * the current vCPU mode is accurate.  in kvm_set_cr3()
1427 	if (!kvm_vcpu_is_legal_cr3(vcpu, cr3))  in kvm_set_cr3()
1430 	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))  in kvm_set_cr3()
1433 	if (cr3 != kvm_read_cr3(vcpu))  in kvm_set_cr3()
1434 		kvm_mmu_new_pgd(vcpu, cr3);  in kvm_set_cr3()
1436 	vcpu->arch.cr3 = cr3;  in kvm_set_cr3()
1437 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_set_cr3()
1449 		kvm_invalidate_pcid(vcpu, pcid);  in kvm_set_cr3()
1455 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)  in kvm_set_cr8()  argument
1459 	if (lapic_in_kernel(vcpu))  in kvm_set_cr8()
1460 		kvm_lapic_set_tpr(vcpu, cr8);  in kvm_set_cr8()
1462 		vcpu->arch.cr8 = cr8;  in kvm_set_cr8()
1467 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)  in kvm_get_cr8()  argument
1469 	if (lapic_in_kernel(vcpu))  in kvm_get_cr8()
1470 		return kvm_lapic_get_cr8(vcpu);  in kvm_get_cr8()
1472 		return vcpu->arch.cr8;  in kvm_get_cr8()
1476 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)  in kvm_update_dr0123()  argument
1480 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {  in kvm_update_dr0123()
1482 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_update_dr0123()
1486 void kvm_update_dr7(struct kvm_vcpu *vcpu)  in kvm_update_dr7()  argument
1490 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)  in kvm_update_dr7()
1491 		dr7 = vcpu->arch.guest_debug_dr7;  in kvm_update_dr7()
1493 		dr7 = vcpu->arch.dr7;  in kvm_update_dr7()
1494 	kvm_x86_call(set_dr7)(vcpu, dr7);  in kvm_update_dr7()
1495 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1497 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;  in kvm_update_dr7()
1501 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)  in kvm_dr6_fixed()  argument
1505 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM))  in kvm_dr6_fixed()
1508 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))  in kvm_dr6_fixed()
1513 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)  in kvm_set_dr()  argument
1515 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_set_dr()
1519 		vcpu->arch.db[array_index_nospec(dr, size)] = val;  in kvm_set_dr()
1520 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))  in kvm_set_dr()
1521 			vcpu->arch.eff_db[dr] = val;  in kvm_set_dr()
1527 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);  in kvm_set_dr()
1533 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;  in kvm_set_dr()
1534 		kvm_update_dr7(vcpu);  in kvm_set_dr()
1542 unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr)  in kvm_get_dr()  argument
1544 	size_t size = ARRAY_SIZE(vcpu->arch.db);  in kvm_get_dr()
1548 		return vcpu->arch.db[array_index_nospec(dr, size)];  in kvm_get_dr()
1551 		return vcpu->arch.dr6;  in kvm_get_dr()
1554 		return vcpu->arch.dr7;  in kvm_get_dr()
1559 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)  in kvm_emulate_rdpmc()  argument
1561 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdpmc()
1564 	if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {  in kvm_emulate_rdpmc()
1565 		kvm_inject_gp(vcpu, 0);  in kvm_emulate_rdpmc()
1569 	kvm_rax_write(vcpu, (u32)data);  in kvm_emulate_rdpmc()
1570 	kvm_rdx_write(vcpu, data >> 32);  in kvm_emulate_rdpmc()
1571 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_rdpmc()
1652 static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in kvm_get_feature_msr()  argument
1676 static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_feature_msr()  argument
1678 	return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R,  in do_get_feature_msr()
1682 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in __kvm_valid_efer()  argument
1684 	if (efer & EFER_AUTOIBRS && !guest_cpu_cap_has(vcpu, X86_FEATURE_AUTOIBRS))  in __kvm_valid_efer()
1687 	if (efer & EFER_FFXSR && !guest_cpu_cap_has(vcpu, X86_FEATURE_FXSR_OPT))  in __kvm_valid_efer()
1690 	if (efer & EFER_SVME && !guest_cpu_cap_has(vcpu, X86_FEATURE_SVM))  in __kvm_valid_efer()
1694 	    !guest_cpu_cap_has(vcpu, X86_FEATURE_LM))  in __kvm_valid_efer()
1697 	if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX))  in __kvm_valid_efer()
1703 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)  in kvm_valid_efer()  argument
1708 	return __kvm_valid_efer(vcpu, efer);  in kvm_valid_efer()
1712 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_efer()  argument
1714 	u64 old_efer = vcpu->arch.efer;  in set_efer()
1722 		if (!__kvm_valid_efer(vcpu, efer))  in set_efer()
1725 		if (is_paging(vcpu) &&  in set_efer()
1726 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))  in set_efer()
1731 	efer |= vcpu->arch.efer & EFER_LMA;  in set_efer()
1733 	r = kvm_x86_call(set_efer)(vcpu, efer);  in set_efer()
1740 		kvm_mmu_reset_context(vcpu);  in set_efer()
1744 		kvm_hv_xsaves_xsavec_maybe_warn(vcpu);  in set_efer()
1755 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)  in kvm_msr_allowed()  argument
1759 	struct kvm *kvm = vcpu->kvm;  in kvm_msr_allowed()
1804 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,  in __kvm_set_msr()  argument
1815 		if (is_noncanonical_msr_address(data, vcpu))  in __kvm_set_msr()
1839 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_set_msr()
1840 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID))  in __kvm_set_msr()
1852 		if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0)  in __kvm_set_msr()
1863 	return kvm_x86_call(set_msr)(vcpu, &msr);  in __kvm_set_msr()
1866 static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in _kvm_set_msr()  argument
1869 	return __kvm_set_msr(vcpu, index, *data, host_initiated);  in _kvm_set_msr()
1872 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_set_msr_ignored_check()  argument
1875 	return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W,  in kvm_set_msr_ignored_check()
1885 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,  in __kvm_get_msr()  argument
1897 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&  in __kvm_get_msr()
1898 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID))  in __kvm_get_msr()
1906 	ret = kvm_x86_call(get_msr)(vcpu, &msr);  in __kvm_get_msr()
1912 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,  in kvm_get_msr_ignored_check()  argument
1915 	return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R,  in kvm_get_msr_ignored_check()
1919 int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr_with_filter()  argument
1921 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))  in kvm_get_msr_with_filter()
1923 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr_with_filter()
1927 int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr_with_filter()  argument
1929 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))  in kvm_set_msr_with_filter()
1931 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr_with_filter()
1935 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)  in kvm_get_msr()  argument
1937 	return kvm_get_msr_ignored_check(vcpu, index, data, false);  in kvm_get_msr()
1941 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)  in kvm_set_msr()  argument
1943 	return kvm_set_msr_ignored_check(vcpu, index, data, false);  in kvm_set_msr()
1947 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)  in complete_userspace_rdmsr()  argument
1949 	if (!vcpu->run->msr.error) {  in complete_userspace_rdmsr()
1950 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);  in complete_userspace_rdmsr()
1951 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);  in complete_userspace_rdmsr()
1955 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)  in complete_emulated_msr_access()  argument
1957 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);  in complete_emulated_msr_access()
1960 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)  in complete_emulated_rdmsr()  argument
1962 	complete_userspace_rdmsr(vcpu);  in complete_emulated_rdmsr()
1963 	return complete_emulated_msr_access(vcpu);  in complete_emulated_rdmsr()
1966 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)  in complete_fast_msr_access()  argument
1968 	return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error);  in complete_fast_msr_access()
1971 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)  in complete_fast_rdmsr()  argument
1973 	complete_userspace_rdmsr(vcpu);  in complete_fast_rdmsr()
1974 	return complete_fast_msr_access(vcpu);  in complete_fast_rdmsr()
1989 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,  in kvm_msr_user_space()  argument
1991 			      int (*completion)(struct kvm_vcpu *vcpu),  in kvm_msr_user_space()  argument
1997 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))  in kvm_msr_user_space()
2000 	vcpu->run->exit_reason = exit_reason;  in kvm_msr_user_space()
2001 	vcpu->run->msr.error = 0;  in kvm_msr_user_space()
2002 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));  in kvm_msr_user_space()
2003 	vcpu->run->msr.reason = msr_reason;  in kvm_msr_user_space()
2004 	vcpu->run->msr.index = index;  in kvm_msr_user_space()
2005 	vcpu->run->msr.data = data;  in kvm_msr_user_space()
2006 	vcpu->arch.complete_userspace_io = completion;  in kvm_msr_user_space()
2011 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_rdmsr()  argument
2013 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_rdmsr()
2017 	r = kvm_get_msr_with_filter(vcpu, ecx, &data);  in kvm_emulate_rdmsr()
2022 		kvm_rax_write(vcpu, data & -1u);  in kvm_emulate_rdmsr()
2023 		kvm_rdx_write(vcpu, (data >> 32) & -1u);  in kvm_emulate_rdmsr()
2026 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,  in kvm_emulate_rdmsr()
2032 	return kvm_x86_call(complete_emulated_msr)(vcpu, r);  in kvm_emulate_rdmsr()
2036 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)  in kvm_emulate_wrmsr()  argument
2038 	u32 ecx = kvm_rcx_read(vcpu);  in kvm_emulate_wrmsr()
2039 	u64 data = kvm_read_edx_eax(vcpu);  in kvm_emulate_wrmsr()
2042 	r = kvm_set_msr_with_filter(vcpu, ecx, data);  in kvm_emulate_wrmsr()
2048 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,  in kvm_emulate_wrmsr()
2057 	return kvm_x86_call(complete_emulated_msr)(vcpu, r);  in kvm_emulate_wrmsr()
2061 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)  in kvm_emulate_as_nop()  argument
2063 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_as_nop()
2066 int kvm_emulate_invd(struct kvm_vcpu *vcpu)  in kvm_emulate_invd()  argument
2069 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_invd()
2073 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)  in kvm_handle_invalid_op()  argument
2075 	kvm_queue_exception(vcpu, UD_VECTOR);  in kvm_handle_invalid_op()
2081 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)  in kvm_emulate_monitor_mwait()  argument
2083 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&  in kvm_emulate_monitor_mwait()
2084 	    !guest_cpu_cap_has(vcpu, X86_FEATURE_MWAIT))  in kvm_emulate_monitor_mwait()
2085 		return kvm_handle_invalid_op(vcpu);  in kvm_emulate_monitor_mwait()
2088 	return kvm_emulate_as_nop(vcpu);  in kvm_emulate_monitor_mwait()
2090 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)  in kvm_emulate_mwait()  argument
2092 	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");  in kvm_emulate_mwait()
2096 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)  in kvm_emulate_monitor()  argument
2098 	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");  in kvm_emulate_monitor()
2102 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)  in kvm_vcpu_exit_request()  argument
2106 	return READ_ONCE(vcpu->mode) == EXITING_GUEST_MODE ||  in kvm_vcpu_exit_request()
2107 	       kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending();  in kvm_vcpu_exit_request()
2117 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_x2apic_icr_irqoff()  argument
2119 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))  in handle_fastpath_set_x2apic_icr_irqoff()
2126 		return kvm_x2apic_icr_write(vcpu->arch.apic, data);  in handle_fastpath_set_x2apic_icr_irqoff()
2131 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)  in handle_fastpath_set_tscdeadline()  argument
2133 	if (!kvm_can_use_hv_timer(vcpu))  in handle_fastpath_set_tscdeadline()
2136 	kvm_set_lapic_tscdeadline_msr(vcpu, data);  in handle_fastpath_set_tscdeadline()
2140 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)  in handle_fastpath_set_msr_irqoff()  argument
2142 	u32 msr = kvm_rcx_read(vcpu);  in handle_fastpath_set_msr_irqoff()
2147 	kvm_vcpu_srcu_read_lock(vcpu);  in handle_fastpath_set_msr_irqoff()
2151 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2152 		handled = !handle_fastpath_set_x2apic_icr_irqoff(vcpu, data);  in handle_fastpath_set_msr_irqoff()
2155 		data = kvm_read_edx_eax(vcpu);  in handle_fastpath_set_msr_irqoff()
2156 		handled = !handle_fastpath_set_tscdeadline(vcpu, data);  in handle_fastpath_set_msr_irqoff()
2164 		if (!kvm_skip_emulated_instruction(vcpu))  in handle_fastpath_set_msr_irqoff()
2173 	kvm_vcpu_srcu_read_unlock(vcpu);  in handle_fastpath_set_msr_irqoff()
2182 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_get_msr()  argument
2184 	return kvm_get_msr_ignored_check(vcpu, index, data, true);  in do_get_msr()
2187 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)  in do_set_msr()  argument
2193 	 * not support modifying the guest vCPU model on the fly, e.g. changing  in do_set_msr()
2198 	if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index) &&  in do_set_msr()
2199 	    (do_get_msr(vcpu, index, &val) || *data != val))  in do_set_msr()
2202 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);  in do_set_msr()
2312 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,  in kvm_write_system_time()  argument
2315 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_write_system_time()
2317 	if (vcpu->vcpu_id == 0 && !host_initiated) {  in kvm_write_system_time()
2319 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2324 	vcpu->arch.time = system_time;  in kvm_write_system_time()
2325 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_write_system_time()
2329 		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,  in kvm_write_system_time()
2332 		kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvm_write_system_time()
2385 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2387 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)  in set_tsc_khz()  argument
2393 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in set_tsc_khz()
2400 			vcpu->arch.tsc_catchup = 1;  in set_tsc_khz()
2401 			vcpu->arch.tsc_always_catchup = 1;  in set_tsc_khz()
2419 	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);  in set_tsc_khz()
2423 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)  in kvm_set_tsc_khz()  argument
2431 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);  in kvm_set_tsc_khz()
2437 			   &vcpu->arch.virtual_tsc_shift,  in kvm_set_tsc_khz()
2438 			   &vcpu->arch.virtual_tsc_mult);  in kvm_set_tsc_khz()
2439 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;  in kvm_set_tsc_khz()
2454 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);  in kvm_set_tsc_khz()
2457 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)  in compute_guest_tsc()  argument
2459 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,  in compute_guest_tsc()
2460 				      vcpu->arch.virtual_tsc_mult,  in compute_guest_tsc()
2461 				      vcpu->arch.virtual_tsc_shift);  in compute_guest_tsc()
2462 	tsc += vcpu->arch.this_tsc_write;  in compute_guest_tsc()
2473 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)  in kvm_track_tsc_matching()  argument
2476 	struct kvm_arch *ka = &vcpu->kvm->arch;  in kvm_track_tsc_matching()
2482 	 * vCPUs doesn't include the reference vCPU, hence "+1".  in kvm_track_tsc_matching()
2485 				 atomic_read(&vcpu->kvm->online_vcpus)) &&  in kvm_track_tsc_matching()
2496 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_track_tsc_matching()
2498 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,  in kvm_track_tsc_matching()
2499 			    atomic_read(&vcpu->kvm->online_vcpus),  in kvm_track_tsc_matching()
2529 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)  in kvm_compute_l1_tsc_offset()  argument
2533 	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);  in kvm_compute_l1_tsc_offset()
2538 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)  in kvm_read_l1_tsc()  argument
2540 	return vcpu->arch.l1_tsc_offset +  in kvm_read_l1_tsc()
2541 		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);  in kvm_read_l1_tsc()
2570 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)  in kvm_vcpu_write_tsc_offset()  argument
2572 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,  in kvm_vcpu_write_tsc_offset()
2573 				   vcpu->arch.l1_tsc_offset,  in kvm_vcpu_write_tsc_offset()
2576 	vcpu->arch.l1_tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2583 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_offset()
2584 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(  in kvm_vcpu_write_tsc_offset()
2586 			kvm_x86_call(get_l2_tsc_offset)(vcpu),  in kvm_vcpu_write_tsc_offset()
2587 			kvm_x86_call(get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_offset()
2589 		vcpu->arch.tsc_offset = l1_offset;  in kvm_vcpu_write_tsc_offset()
2591 	kvm_x86_call(write_tsc_offset)(vcpu);  in kvm_vcpu_write_tsc_offset()
2594 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)  in kvm_vcpu_write_tsc_multiplier()  argument
2596 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2599 	if (is_guest_mode(vcpu))  in kvm_vcpu_write_tsc_multiplier()
2600 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(  in kvm_vcpu_write_tsc_multiplier()
2602 			kvm_x86_call(get_l2_tsc_multiplier)(vcpu));  in kvm_vcpu_write_tsc_multiplier()
2604 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;  in kvm_vcpu_write_tsc_multiplier()
2607 		kvm_x86_call(write_tsc_multiplier)(vcpu);  in kvm_vcpu_write_tsc_multiplier()
2625  * offset for the vcpu and tracks the TSC matching generation that the vcpu
2628 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,  in __kvm_synchronize_tsc()  argument
2631 	struct kvm *kvm = vcpu->kvm;  in __kvm_synchronize_tsc()
2641 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;  in __kvm_synchronize_tsc()
2644 	vcpu->arch.last_guest_tsc = tsc;  in __kvm_synchronize_tsc()
2646 	kvm_vcpu_write_tsc_offset(vcpu, offset);  in __kvm_synchronize_tsc()
2663 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {  in __kvm_synchronize_tsc()
2667 	/* Keep track of which generation this VCPU has synchronized to */  in __kvm_synchronize_tsc()
2668 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;  in __kvm_synchronize_tsc()
2669 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;  in __kvm_synchronize_tsc()
2670 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;  in __kvm_synchronize_tsc()
2672 	kvm_track_tsc_matching(vcpu, !matched);  in __kvm_synchronize_tsc()
2675 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)  in kvm_synchronize_tsc()  argument
2678 	struct kvm *kvm = vcpu->kvm;  in kvm_synchronize_tsc()
2685 	offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2689 	if (vcpu->arch.virtual_tsc_khz) {  in kvm_synchronize_tsc()
2692 			 * Force synchronization when creating a vCPU, or when  in kvm_synchronize_tsc()
2698 						nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2699 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;  in kvm_synchronize_tsc()
2703 			 * previously set vCPU, we assume that they were intended to be  in kvm_synchronize_tsc()
2710 			 * value written by userspace (on any vCPU) should not be subject  in kvm_synchronize_tsc()
2712 			 * come from the kernel's default vCPU creation. Make the 1-second  in kvm_synchronize_tsc()
2730 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {  in kvm_synchronize_tsc()
2734 			u64 delta = nsec_to_cycles(vcpu, elapsed);  in kvm_synchronize_tsc()
2736 			offset = kvm_compute_l1_tsc_offset(vcpu, data);  in kvm_synchronize_tsc()
2741 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched);  in kvm_synchronize_tsc()
2745 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,  in adjust_tsc_offset_guest()  argument
2748 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;  in adjust_tsc_offset_guest()
2749 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);  in adjust_tsc_offset_guest()
2752 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)  in adjust_tsc_offset_host()  argument
2754 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)  in adjust_tsc_offset_host()
2757 				   vcpu->arch.l1_tsc_scaling_ratio);  in adjust_tsc_offset_host()
2758 	adjust_tsc_offset_guest(vcpu, adjustment);  in adjust_tsc_offset_host()
3027 	struct kvm_vcpu *vcpu;  in kvm_end_pvclock_update()  local
3032 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3033 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_end_pvclock_update()
3036 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_end_pvclock_update()
3037 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);  in kvm_end_pvclock_update()
3124 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_setup_guest_pvclock()  local
3141 	 * This VCPU is paused, but it's legal for a guest to read another  in kvm_setup_guest_pvclock()
3142 	 * VCPU's kvmclock, so we really have to follow the specification where  in kvm_setup_guest_pvclock()
3147 	guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;  in kvm_setup_guest_pvclock()
3151 	vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);  in kvm_setup_guest_pvclock()
3153 	if (vcpu->pvclock_set_guest_stopped_request) {  in kvm_setup_guest_pvclock()
3154 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;  in kvm_setup_guest_pvclock()
3155 		vcpu->pvclock_set_guest_stopped_request = false;  in kvm_setup_guest_pvclock()
3158 	memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));  in kvm_setup_guest_pvclock()
3165 	guest_hv_clock->version = ++vcpu->hv_clock.version;  in kvm_setup_guest_pvclock()
3170 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);  in kvm_setup_guest_pvclock()
3177 	struct kvm_vcpu_arch *vcpu = &v->arch;  in kvm_guest_time_update()  local
3229 	 *   2) Broken TSC compensation resets the base at each VCPU  in kvm_guest_time_update()
3235 	if (vcpu->tsc_catchup) {  in kvm_guest_time_update()
3251 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {  in kvm_guest_time_update()
3253 				   &vcpu->hv_clock.tsc_shift,  in kvm_guest_time_update()
3254 				   &vcpu->hv_clock.tsc_to_system_mul);  in kvm_guest_time_update()
3255 		vcpu->hw_tsc_khz = tgt_tsc_khz;  in kvm_guest_time_update()
3259 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;  in kvm_guest_time_update()
3260 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;  in kvm_guest_time_update()
3261 	vcpu->last_guest_tsc = tsc_timestamp;  in kvm_guest_time_update()
3268 	vcpu->hv_clock.flags = pvclock_flags;  in kvm_guest_time_update()
3270 	if (vcpu->pv_time.active)  in kvm_guest_time_update()
3271 		kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0, false);  in kvm_guest_time_update()
3273 	if (vcpu->xen.vcpu_info_cache.active)  in kvm_guest_time_update()
3274 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,  in kvm_guest_time_update()
3277 	if (vcpu->xen.vcpu_time_info_cache.active)  in kvm_guest_time_update()
3278 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0,  in kvm_guest_time_update()
3281 	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);  in kvm_guest_time_update()
3362  * kvmclock updates which are isolated to a given vcpu, such as
3363  * vcpu->cpu migration, should not allow system_timestamp from
3365  * correction applies to one vcpu's system_timestamp but not
3371  * The time for a remote vcpu to update its kvmclock is bound
3384 	struct kvm_vcpu *vcpu;  in kvmclock_update_fn()  local
3386 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvmclock_update_fn()
3387 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvmclock_update_fn()
3388 		kvm_vcpu_kick(vcpu);  in kvmclock_update_fn()
3428 static bool can_set_mci_status(struct kvm_vcpu *vcpu)  in can_set_mci_status()  argument
3431 	if (guest_cpuid_is_amd_compatible(vcpu))  in can_set_mci_status()
3432 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));  in can_set_mci_status()
3437 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in set_msr_mce()  argument
3439 	u64 mcg_cap = vcpu->arch.mcg_cap;  in set_msr_mce()
3447 		vcpu->arch.mcg_status = data;  in set_msr_mce()
3455 		vcpu->arch.mcg_ctl = data;  in set_msr_mce()
3469 		vcpu->arch.mci_ctl2_banks[offset] = data;  in set_msr_mce()
3496 		    data != 0 && !can_set_mci_status(vcpu))  in set_msr_mce()
3501 		vcpu->arch.mce_banks[offset] = data;  in set_msr_mce()
3509 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)  in kvm_pv_async_pf_enabled()  argument
3513 	return (vcpu->arch.apf.msr_en_val & mask) == mask;  in kvm_pv_async_pf_enabled()
3516 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf()  argument
3524 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&  in kvm_pv_enable_async_pf()
3528 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&  in kvm_pv_enable_async_pf()
3532 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf()
3535 	vcpu->arch.apf.msr_en_val = data;  in kvm_pv_enable_async_pf()
3537 	if (!kvm_pv_async_pf_enabled(vcpu)) {  in kvm_pv_enable_async_pf()
3538 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_pv_enable_async_pf()
3539 		kvm_async_pf_hash_reset(vcpu);  in kvm_pv_enable_async_pf()
3543 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,  in kvm_pv_enable_async_pf()
3547 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);  in kvm_pv_enable_async_pf()
3548 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;  in kvm_pv_enable_async_pf()
3550 	kvm_async_pf_wakeup_all(vcpu);  in kvm_pv_enable_async_pf()
3555 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)  in kvm_pv_enable_async_pf_int()  argument
3561 	if (!lapic_in_kernel(vcpu))  in kvm_pv_enable_async_pf_int()
3564 	vcpu->arch.apf.msr_int_val = data;  in kvm_pv_enable_async_pf_int()
3566 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;  in kvm_pv_enable_async_pf_int()
3571 static void kvmclock_reset(struct kvm_vcpu *vcpu)  in kvmclock_reset()  argument
3573 	kvm_gpc_deactivate(&vcpu->arch.pv_time);  in kvmclock_reset()
3574 	vcpu->arch.time = 0;  in kvmclock_reset()
3577 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_all()  argument
3579 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_all()
3580 	kvm_x86_call(flush_tlb_all)(vcpu);  in kvm_vcpu_flush_tlb_all()
3583 	kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);  in kvm_vcpu_flush_tlb_all()
3586 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_guest()  argument
3588 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_guest()
3597 		kvm_mmu_sync_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3598 		kvm_mmu_sync_prev_roots(vcpu);  in kvm_vcpu_flush_tlb_guest()
3601 	kvm_x86_call(flush_tlb_guest)(vcpu);  in kvm_vcpu_flush_tlb_guest()
3607 	kvm_hv_vcpu_purge_flush_tlb(vcpu);  in kvm_vcpu_flush_tlb_guest()
3611 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)  in kvm_vcpu_flush_tlb_current()  argument
3613 	++vcpu->stat.tlb_flush;  in kvm_vcpu_flush_tlb_current()
3614 	kvm_x86_call(flush_tlb_current)(vcpu);  in kvm_vcpu_flush_tlb_current()
3623 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)  in kvm_service_local_tlb_flush_requests()  argument
3625 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))  in kvm_service_local_tlb_flush_requests()
3626 		kvm_vcpu_flush_tlb_current(vcpu);  in kvm_service_local_tlb_flush_requests()
3628 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))  in kvm_service_local_tlb_flush_requests()
3629 		kvm_vcpu_flush_tlb_guest(vcpu);  in kvm_service_local_tlb_flush_requests()
3633 static void record_steal_time(struct kvm_vcpu *vcpu)  in record_steal_time()  argument
3635 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in record_steal_time()
3638 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in record_steal_time()
3642 	if (kvm_xen_msr_enabled(vcpu->kvm)) {  in record_steal_time()
3643 		kvm_xen_runstate_set_running(vcpu);  in record_steal_time()
3647 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in record_steal_time()
3650 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))  in record_steal_time()
3653 	slots = kvm_memslots(vcpu->kvm);  in record_steal_time()
3661 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||  in record_steal_time()
3671 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {  in record_steal_time()
3690 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3692 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,  in record_steal_time()
3695 			kvm_vcpu_flush_tlb_guest(vcpu);  in record_steal_time()
3704 		vcpu->arch.st.preempted = 0;  in record_steal_time()
3718 		vcpu->arch.st.last_steal;  in record_steal_time()
3719 	vcpu->arch.st.last_steal = current->sched_info.run_delay;  in record_steal_time()
3728 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in record_steal_time()
3731 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_set_msr_common()  argument
3736 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)  in kvm_set_msr_common()
3737 		return kvm_xen_write_hypercall_page(vcpu, data);  in kvm_set_msr_common()
3752 			vcpu->arch.microcode_version = data;  in kvm_set_msr_common()
3756 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))  in kvm_set_msr_common()
3758 		vcpu->arch.arch_capabilities = data;  in kvm_set_msr_common()
3762 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))  in kvm_set_msr_common()
3770 		 * disallows changing feature MSRs after the vCPU has run; PMU  in kvm_set_msr_common()
3771 		 * refresh will bug the VM if called after the vCPU has run.  in kvm_set_msr_common()
3773 		if (vcpu->arch.perf_capabilities == data)  in kvm_set_msr_common()
3776 		vcpu->arch.perf_capabilities = data;  in kvm_set_msr_common()
3777 		kvm_pmu_refresh(vcpu);  in kvm_set_msr_common()
3783 			if ((!guest_has_pred_cmd_msr(vcpu)))  in kvm_set_msr_common()
3786 			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SPEC_CTRL) &&  in kvm_set_msr_common()
3787 			    !guest_cpu_cap_has(vcpu, X86_FEATURE_AMD_IBPB))  in kvm_set_msr_common()
3790 			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SBPB))  in kvm_set_msr_common()
3811 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D))  in kvm_set_msr_common()
3822 		return set_efer(vcpu, msr_info);  in kvm_set_msr_common()
3834 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3837 		vcpu->arch.msr_hwcr = data;  in kvm_set_msr_common()
3841 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
3849 		vcpu->arch.pat = data;  in kvm_set_msr_common()
3853 		return kvm_mtrr_set_msr(vcpu, msr, data);  in kvm_set_msr_common()
3855 		return kvm_apic_set_base(vcpu, data, msr_info->host_initiated);  in kvm_set_msr_common()
3857 		return kvm_x2apic_msr_write(vcpu, msr, data);  in kvm_set_msr_common()
3859 		kvm_set_lapic_tscdeadline_msr(vcpu, data);  in kvm_set_msr_common()
3862 		if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSC_ADJUST)) {  in kvm_set_msr_common()
3864 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;  in kvm_set_msr_common()
3865 				adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3869 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_msr_common()
3871 			vcpu->arch.ia32_tsc_adjust_msr = data;  in kvm_set_msr_common()
3875 		u64 old_val = vcpu->arch.ia32_misc_enable_msr;  in kvm_set_msr_common()
3887 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&  in kvm_set_msr_common()
3889 			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XMM3))  in kvm_set_msr_common()
3891 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3892 			kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3894 			vcpu->arch.ia32_misc_enable_msr = data;  in kvm_set_msr_common()
3901 		vcpu->arch.smbase = data;  in kvm_set_msr_common()
3904 		vcpu->arch.msr_ia32_power_ctl = data;  in kvm_set_msr_common()
3908 			kvm_synchronize_tsc(vcpu, &data);  in kvm_set_msr_common()
3910 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;  in kvm_set_msr_common()
3911 			adjust_tsc_offset_guest(vcpu, adj);  in kvm_set_msr_common()
3912 			vcpu->arch.ia32_tsc_adjust_msr += adj;  in kvm_set_msr_common()
3917 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_set_msr_common()
3926 		vcpu->arch.ia32_xss = data;  in kvm_set_msr_common()
3927 		kvm_update_cpuid_runtime(vcpu);  in kvm_set_msr_common()
3932 		vcpu->arch.smi_count = data;  in kvm_set_msr_common()
3935 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3938 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3939 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3942 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3945 		vcpu->kvm->arch.wall_clock = data;  in kvm_set_msr_common()
3946 		kvm_write_wall_clock(vcpu->kvm, data, 0);  in kvm_set_msr_common()
3949 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_set_msr_common()
3952 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);  in kvm_set_msr_common()
3955 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_set_msr_common()
3958 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);  in kvm_set_msr_common()
3961 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_set_msr_common()
3964 		if (kvm_pv_enable_async_pf(vcpu, data))  in kvm_set_msr_common()
3968 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3971 		if (kvm_pv_enable_async_pf_int(vcpu, data))  in kvm_set_msr_common()
3975 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_set_msr_common()
3978 			vcpu->arch.apf.pageready_pending = false;  in kvm_set_msr_common()
3979 			kvm_check_async_pf_completion(vcpu);  in kvm_set_msr_common()
3983 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_set_msr_common()
3992 		vcpu->arch.st.msr_val = data;  in kvm_set_msr_common()
3997 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_set_msr_common()
4001 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_set_msr_common()
4004 		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))  in kvm_set_msr_common()
4009 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_set_msr_common()
4016 		vcpu->arch.msr_kvm_poll_control = data;  in kvm_set_msr_common()
4023 		return set_msr_mce(vcpu, msr_info);  in kvm_set_msr_common()
4029 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
4030 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
4033 			kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
4056 		return kvm_hv_set_msr_common(vcpu, msr, data,  in kvm_set_msr_common()
4063 		kvm_pr_unimpl_wrmsr(vcpu, msr, data);  in kvm_set_msr_common()
4066 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
4068 		vcpu->arch.osvw.length = data;  in kvm_set_msr_common()
4071 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))  in kvm_set_msr_common()
4073 		vcpu->arch.osvw.status = data;  in kvm_set_msr_common()
4078 		vcpu->arch.msr_platform_info = data;  in kvm_set_msr_common()
4083 		     !supports_cpuid_fault(vcpu)))  in kvm_set_msr_common()
4085 		vcpu->arch.msr_misc_features_enables = data;  in kvm_set_msr_common()
4090 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
4093 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
4096 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);  in kvm_set_msr_common()
4100 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))  in kvm_set_msr_common()
4103 		if (data & ~kvm_guest_supported_xfd(vcpu))  in kvm_set_msr_common()
4106 		vcpu->arch.guest_fpu.xfd_err = data;  in kvm_set_msr_common()
4110 		if (kvm_pmu_is_valid_msr(vcpu, msr))  in kvm_set_msr_common()
4111 			return kvm_pmu_set_msr(vcpu, msr_info);  in kvm_set_msr_common()
4119 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)  in get_msr_mce()  argument
4122 	u64 mcg_cap = vcpu->arch.mcg_cap;  in get_msr_mce()
4132 		data = vcpu->arch.mcg_cap;  in get_msr_mce()
4137 		data = vcpu->arch.mcg_ctl;  in get_msr_mce()
4140 		data = vcpu->arch.mcg_status;  in get_msr_mce()
4151 		data = vcpu->arch.mci_ctl2_banks[offset];  in get_msr_mce()
4160 		data = vcpu->arch.mce_banks[offset];  in get_msr_mce()
4169 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)  in kvm_get_msr_common()  argument
4207 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4208 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4212 		msr_info->data = vcpu->arch.microcode_version;  in kvm_get_msr_common()
4215 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))  in kvm_get_msr_common()
4217 		msr_info->data = vcpu->arch.arch_capabilities;  in kvm_get_msr_common()
4220 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))  in kvm_get_msr_common()
4222 		msr_info->data = vcpu->arch.perf_capabilities;  in kvm_get_msr_common()
4225 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;  in kvm_get_msr_common()
4240 			offset = vcpu->arch.l1_tsc_offset;  in kvm_get_msr_common()
4241 			ratio = vcpu->arch.l1_tsc_scaling_ratio;  in kvm_get_msr_common()
4243 			offset = vcpu->arch.tsc_offset;  in kvm_get_msr_common()
4244 			ratio = vcpu->arch.tsc_scaling_ratio;  in kvm_get_msr_common()
4251 		msr_info->data = vcpu->arch.pat;  in kvm_get_msr_common()
4256 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4275 		msr_info->data = vcpu->arch.apic_base;  in kvm_get_msr_common()
4278 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);  in kvm_get_msr_common()
4280 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);  in kvm_get_msr_common()
4283 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;  in kvm_get_msr_common()
4286 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;  in kvm_get_msr_common()
4291 		msr_info->data = vcpu->arch.smbase;  in kvm_get_msr_common()
4294 		msr_info->data = vcpu->arch.smi_count;  in kvm_get_msr_common()
4303 		msr_info->data = vcpu->arch.efer;  in kvm_get_msr_common()
4306 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4309 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4312 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4315 		msr_info->data = vcpu->kvm->arch.wall_clock;  in kvm_get_msr_common()
4318 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))  in kvm_get_msr_common()
4321 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4324 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))  in kvm_get_msr_common()
4327 		msr_info->data = vcpu->arch.time;  in kvm_get_msr_common()
4330 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))  in kvm_get_msr_common()
4333 		msr_info->data = vcpu->arch.apf.msr_en_val;  in kvm_get_msr_common()
4336 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4339 		msr_info->data = vcpu->arch.apf.msr_int_val;  in kvm_get_msr_common()
4342 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))  in kvm_get_msr_common()
4348 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))  in kvm_get_msr_common()
4351 		msr_info->data = vcpu->arch.st.msr_val;  in kvm_get_msr_common()
4354 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))  in kvm_get_msr_common()
4357 		msr_info->data = vcpu->arch.pv_eoi.msr_val;  in kvm_get_msr_common()
4360 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))  in kvm_get_msr_common()
4363 		msr_info->data = vcpu->arch.msr_kvm_poll_control;  in kvm_get_msr_common()
4372 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,  in kvm_get_msr_common()
4376 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))  in kvm_get_msr_common()
4378 		msr_info->data = vcpu->arch.ia32_xss;  in kvm_get_msr_common()
4403 		return kvm_hv_get_msr_common(vcpu,  in kvm_get_msr_common()
4421 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4423 		msr_info->data = vcpu->arch.osvw.length;  in kvm_get_msr_common()
4426 		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))  in kvm_get_msr_common()
4428 		msr_info->data = vcpu->arch.osvw.status;  in kvm_get_msr_common()
4432 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)  in kvm_get_msr_common()
4434 		msr_info->data = vcpu->arch.msr_platform_info;  in kvm_get_msr_common()
4437 		msr_info->data = vcpu->arch.msr_misc_features_enables;  in kvm_get_msr_common()
4440 		msr_info->data = vcpu->arch.msr_hwcr;  in kvm_get_msr_common()
4445 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4448 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;  in kvm_get_msr_common()
4452 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))  in kvm_get_msr_common()
4455 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;  in kvm_get_msr_common()
4459 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))  in kvm_get_msr_common()
4460 			return kvm_pmu_get_msr(vcpu, msr_info);  in kvm_get_msr_common()
4473 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,  in __msr_io()  argument
4475 		    int (*do_msr)(struct kvm_vcpu *vcpu,  in __msr_io()  argument
4481 		if (do_msr(vcpu, entries[i].index, &entries[i].data))  in __msr_io()
4492 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,  in msr_io()  argument
4493 		  int (*do_msr)(struct kvm_vcpu *vcpu,  in msr_io()  argument
4517 	r = __msr_io(vcpu, &msrs, entries, do_msr);  in msr_io()
4549 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,  in kvm_ioctl_get_supported_hv_cpuid()  argument
4559 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_ioctl_get_supported_hv_cpuid()
4947 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)  in need_emulate_wbinvd()  argument
4949 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);  in need_emulate_wbinvd()
4952 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_vcpu_load()  argument
4954 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);  in kvm_arch_vcpu_load()
4956 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_arch_vcpu_load()
4958 	if (vcpu->scheduled_out && pmu->version && pmu->event_count) {  in kvm_arch_vcpu_load()
4960 		kvm_make_request(KVM_REQ_PMU, vcpu);  in kvm_arch_vcpu_load()
4964 	if (need_emulate_wbinvd(vcpu)) {  in kvm_arch_vcpu_load()
4966 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_load()
4967 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
4968 			smp_call_function_single(vcpu->cpu,  in kvm_arch_vcpu_load()
4972 	kvm_x86_call(vcpu_load)(vcpu, cpu);  in kvm_arch_vcpu_load()
4975 	vcpu->arch.host_pkru = read_pkru();  in kvm_arch_vcpu_load()
4978 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {  in kvm_arch_vcpu_load()
4979 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);  in kvm_arch_vcpu_load()
4980 		vcpu->arch.tsc_offset_adjustment = 0;  in kvm_arch_vcpu_load()
4981 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
4984 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {  in kvm_arch_vcpu_load()
4985 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :  in kvm_arch_vcpu_load()
4986 				rdtsc() - vcpu->arch.last_host_tsc;  in kvm_arch_vcpu_load()
4991 			u64 offset = kvm_compute_l1_tsc_offset(vcpu,  in kvm_arch_vcpu_load()
4992 						vcpu->arch.last_guest_tsc);  in kvm_arch_vcpu_load()
4993 			kvm_vcpu_write_tsc_offset(vcpu, offset);  in kvm_arch_vcpu_load()
4994 			vcpu->arch.tsc_catchup = 1;  in kvm_arch_vcpu_load()
4997 		if (kvm_lapic_hv_timer_in_use(vcpu))  in kvm_arch_vcpu_load()
4998 			kvm_lapic_restart_hv_timer(vcpu);  in kvm_arch_vcpu_load()
5002 		 * kvmclock on vcpu->cpu migration  in kvm_arch_vcpu_load()
5004 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)  in kvm_arch_vcpu_load()
5005 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);  in kvm_arch_vcpu_load()
5006 		if (vcpu->cpu != cpu)  in kvm_arch_vcpu_load()
5007 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);  in kvm_arch_vcpu_load()
5008 		vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
5011 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);  in kvm_arch_vcpu_load()
5014 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)  in kvm_steal_time_set_preempted()  argument
5016 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;  in kvm_steal_time_set_preempted()
5020 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;  in kvm_steal_time_set_preempted()
5023 	 * The vCPU can be marked preempted if and only if the VM-Exit was on  in kvm_steal_time_set_preempted()
5026 	 * when this is true, for example allowing the vCPU to be marked  in kvm_steal_time_set_preempted()
5029 	if (!vcpu->arch.at_instruction_boundary) {  in kvm_steal_time_set_preempted()
5030 		vcpu->stat.preemption_other++;  in kvm_steal_time_set_preempted()
5034 	vcpu->stat.preemption_reported++;  in kvm_steal_time_set_preempted()
5035 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))  in kvm_steal_time_set_preempted()
5038 	if (vcpu->arch.st.preempted)  in kvm_steal_time_set_preempted()
5042 	if (unlikely(current->mm != vcpu->kvm->mm))  in kvm_steal_time_set_preempted()
5045 	slots = kvm_memslots(vcpu->kvm);  in kvm_steal_time_set_preempted()
5056 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;  in kvm_steal_time_set_preempted()
5058 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));  in kvm_steal_time_set_preempted()
5061 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_put()  argument
5065 	if (vcpu->preempted) {  in kvm_arch_vcpu_put()
5071 		vcpu->arch.preempted_in_kernel = vcpu->arch.guest_state_protected ||  in kvm_arch_vcpu_put()
5072 						 !kvm_x86_call(get_cpl_no_cache)(vcpu);  in kvm_arch_vcpu_put()
5078 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_put()
5079 		if (kvm_xen_msr_enabled(vcpu->kvm))  in kvm_arch_vcpu_put()
5080 			kvm_xen_runstate_set_preempted(vcpu);  in kvm_arch_vcpu_put()
5082 			kvm_steal_time_set_preempted(vcpu);  in kvm_arch_vcpu_put()
5083 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_put()
5086 	kvm_x86_call(vcpu_put)(vcpu);  in kvm_arch_vcpu_put()
5087 	vcpu->arch.last_host_tsc = rdtsc();  in kvm_arch_vcpu_put()
5090 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_get_lapic()  argument
5093 	kvm_x86_call(sync_pir_to_irr)(vcpu);  in kvm_vcpu_ioctl_get_lapic()
5095 	return kvm_apic_get_state(vcpu, s);  in kvm_vcpu_ioctl_get_lapic()
5098 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_set_lapic()  argument
5103 	r = kvm_apic_set_state(vcpu, s);  in kvm_vcpu_ioctl_set_lapic()
5106 	update_cr8_intercept(vcpu);  in kvm_vcpu_ioctl_set_lapic()
5111 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)  in kvm_cpu_accept_dm_intr()  argument
5119 	if (kvm_cpu_has_extint(vcpu))  in kvm_cpu_accept_dm_intr()
5123 	return (!lapic_in_kernel(vcpu) ||  in kvm_cpu_accept_dm_intr()
5124 		kvm_apic_accept_pic_intr(vcpu));  in kvm_cpu_accept_dm_intr()
5127 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)  in kvm_vcpu_ready_for_interrupt_injection()  argument
5136 	return (kvm_arch_interrupt_allowed(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
5137 		kvm_cpu_accept_dm_intr(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
5138 		!kvm_event_needs_reinjection(vcpu) &&  in kvm_vcpu_ready_for_interrupt_injection()
5139 		!kvm_is_exception_pending(vcpu));  in kvm_vcpu_ready_for_interrupt_injection()
5142 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_interrupt()  argument
5148 	if (!irqchip_in_kernel(vcpu->kvm)) {  in kvm_vcpu_ioctl_interrupt()
5149 		kvm_queue_interrupt(vcpu, irq->irq, false);  in kvm_vcpu_ioctl_interrupt()
5150 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5158 	if (pic_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_interrupt()
5161 	if (vcpu->arch.pending_external_vector != -1)  in kvm_vcpu_ioctl_interrupt()
5164 	vcpu->arch.pending_external_vector = irq->irq;  in kvm_vcpu_ioctl_interrupt()
5165 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_interrupt()
5169 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)  in kvm_vcpu_ioctl_nmi()  argument
5171 	kvm_inject_nmi(vcpu);  in kvm_vcpu_ioctl_nmi()
5176 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,  in vcpu_ioctl_tpr_access_reporting()  argument
5181 	vcpu->arch.tpr_access_reporting = !!tac->enabled;  in vcpu_ioctl_tpr_access_reporting()
5185 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_setup_mce()  argument
5197 	vcpu->arch.mcg_cap = mcg_cap;  in kvm_vcpu_ioctl_x86_setup_mce()
5200 		vcpu->arch.mcg_ctl = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5203 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;  in kvm_vcpu_ioctl_x86_setup_mce()
5205 			vcpu->arch.mci_ctl2_banks[bank] = 0;  in kvm_vcpu_ioctl_x86_setup_mce()
5208 	kvm_apic_after_set_mcg_cap(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5210 	kvm_x86_call(setup_mce)(vcpu);  in kvm_vcpu_ioctl_x86_setup_mce()
5232 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)  in kvm_vcpu_x86_set_ucna()  argument
5234 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_x86_set_ucna()
5239 	vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_x86_set_ucna()
5242 	    !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))  in kvm_vcpu_x86_set_ucna()
5245 	if (lapic_in_kernel(vcpu))  in kvm_vcpu_x86_set_ucna()
5246 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);  in kvm_vcpu_x86_set_ucna()
5251 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_mce()  argument
5254 	u64 mcg_cap = vcpu->arch.mcg_cap;  in kvm_vcpu_ioctl_x86_set_mce()
5256 	u64 *banks = vcpu->arch.mce_banks;  in kvm_vcpu_ioctl_x86_set_mce()
5264 		return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);  in kvm_vcpu_ioctl_x86_set_mce()
5271 	    vcpu->arch.mcg_ctl != ~(u64)0)  in kvm_vcpu_ioctl_x86_set_mce()
5280 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||  in kvm_vcpu_ioctl_x86_set_mce()
5281 		    !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) {  in kvm_vcpu_ioctl_x86_set_mce()
5282 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_mce()
5289 		vcpu->arch.mcg_status = mce->mcg_status;  in kvm_vcpu_ioctl_x86_set_mce()
5291 		kvm_queue_exception(vcpu, MC_VECTOR);  in kvm_vcpu_ioctl_x86_set_mce()
5304 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_vcpu_events()  argument
5309 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5312 	if (kvm_check_request(KVM_REQ_SMI, vcpu))  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5313 		process_smi(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5323 	if (vcpu->arch.exception_vmexit.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5324 	    !vcpu->arch.exception.pending &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5325 	    !vcpu->arch.exception.injected)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5326 		ex = &vcpu->arch.exception_vmexit;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5328 		ex = &vcpu->arch.exception;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5338 	if (!vcpu->kvm->arch.exception_payload_enabled &&  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5340 		kvm_deliver_exception_payload(vcpu, ex);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5358 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5368 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5369 	events->interrupt.nr = vcpu->arch.interrupt.nr;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5370 	events->interrupt.shadow = kvm_x86_call(get_interrupt_shadow)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5372 	events->nmi.injected = vcpu->arch.nmi_injected;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5373 	events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5374 	events->nmi.masked = kvm_x86_call(get_nmi_mask)(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5379 	events->smi.smm = is_smm(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5380 	events->smi.pending = vcpu->arch.smi_pending;  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5382 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5384 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5389 	if (vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5391 	if (vcpu->kvm->arch.triple_fault_event) {  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5392 		events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_get_vcpu_events()
5397 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()  argument
5409 		if (!vcpu->kvm->arch.exception_payload_enabled)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5427 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5430 	process_nmi(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5440 	vcpu->arch.exception_from_userspace = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5442 	vcpu->arch.exception_vmexit.pending = false;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5444 	vcpu->arch.exception.injected = events->exception.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5445 	vcpu->arch.exception.pending = events->exception.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5446 	vcpu->arch.exception.vector = events->exception.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5447 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5448 	vcpu->arch.exception.error_code = events->exception.error_code;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5449 	vcpu->arch.exception.has_payload = events->exception_has_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5450 	vcpu->arch.exception.payload = events->exception_payload;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5452 	vcpu->arch.interrupt.injected = events->interrupt.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5453 	vcpu->arch.interrupt.nr = events->interrupt.nr;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5454 	vcpu->arch.interrupt.soft = events->interrupt.soft;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5456 		kvm_x86_call(set_interrupt_shadow)(vcpu,  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5459 	vcpu->arch.nmi_injected = events->nmi.injected;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5461 		vcpu->arch.nmi_pending = 0;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5462 		atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5464 			kvm_make_request(KVM_REQ_NMI, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5466 	kvm_x86_call(set_nmi_mask)(vcpu, events->nmi.masked);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5469 	    lapic_in_kernel(vcpu))  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5470 		vcpu->arch.apic->sipi_vector = events->sipi_vector;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5474 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5475 			kvm_leave_nested(vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5476 			kvm_smm_changed(vcpu, events->smi.smm);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5479 		vcpu->arch.smi_pending = events->smi.pending;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5483 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5485 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5494 		if (lapic_in_kernel(vcpu)) {  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5496 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5498 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5503 		if (!vcpu->kvm->arch.triple_fault_event)  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5506 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5508 			kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5511 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_ioctl_x86_set_vcpu_events()
5516 static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_debugregs()  argument
5521 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_vcpu_ioctl_x86_get_debugregs()
5522 	    vcpu->arch.guest_state_protected)  in kvm_vcpu_ioctl_x86_get_debugregs()
5527 	BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db));  in kvm_vcpu_ioctl_x86_get_debugregs()
5528 	for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)  in kvm_vcpu_ioctl_x86_get_debugregs()
5529 		dbgregs->db[i] = vcpu->arch.db[i];  in kvm_vcpu_ioctl_x86_get_debugregs()
5531 	dbgregs->dr6 = vcpu->arch.dr6;  in kvm_vcpu_ioctl_x86_get_debugregs()
5532 	dbgregs->dr7 = vcpu->arch.dr7;  in kvm_vcpu_ioctl_x86_get_debugregs()
5536 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_debugregs()  argument
5541 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_vcpu_ioctl_x86_set_debugregs()
5542 	    vcpu->arch.guest_state_protected)  in kvm_vcpu_ioctl_x86_set_debugregs()
5553 	for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)  in kvm_vcpu_ioctl_x86_set_debugregs()
5554 		vcpu->arch.db[i] = dbgregs->db[i];  in kvm_vcpu_ioctl_x86_set_debugregs()
5556 	kvm_update_dr0123(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5557 	vcpu->arch.dr6 = dbgregs->dr6;  in kvm_vcpu_ioctl_x86_set_debugregs()
5558 	vcpu->arch.dr7 = dbgregs->dr7;  in kvm_vcpu_ioctl_x86_set_debugregs()
5559 	kvm_update_dr7(vcpu);  in kvm_vcpu_ioctl_x86_set_debugregs()
5565 static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave2()  argument
5580 	u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |  in kvm_vcpu_ioctl_x86_get_xsave2()
5583 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_get_xsave2()
5584 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;  in kvm_vcpu_ioctl_x86_get_xsave2()
5586 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,  in kvm_vcpu_ioctl_x86_get_xsave2()
5587 				       supported_xcr0, vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_get_xsave2()
5591 static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xsave()  argument
5594 	return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,  in kvm_vcpu_ioctl_x86_get_xsave()
5598 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xsave()  argument
5601 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_vcpu_ioctl_x86_set_xsave()
5602 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;  in kvm_vcpu_ioctl_x86_set_xsave()
5604 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,  in kvm_vcpu_ioctl_x86_set_xsave()
5607 					      &vcpu->arch.pkru);  in kvm_vcpu_ioctl_x86_set_xsave()
5610 static int kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_get_xcrs()  argument
5613 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_vcpu_ioctl_x86_get_xcrs()
5614 	    vcpu->arch.guest_state_protected)  in kvm_vcpu_ioctl_x86_get_xcrs()
5625 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;  in kvm_vcpu_ioctl_x86_get_xcrs()
5629 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_x86_set_xcrs()  argument
5634 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_vcpu_ioctl_x86_set_xcrs()
5635 	    vcpu->arch.guest_state_protected)  in kvm_vcpu_ioctl_x86_set_xcrs()
5647 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,  in kvm_vcpu_ioctl_x86_set_xcrs()
5662 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)  in kvm_set_guest_paused()  argument
5664 	if (!vcpu->arch.pv_time.active)  in kvm_set_guest_paused()
5666 	vcpu->arch.pvclock_set_guest_stopped_request = true;  in kvm_set_guest_paused()
5667 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_set_guest_paused()
5671 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_has_attr()  argument
5687 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_get_attr()  argument
5696 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))  in kvm_arch_tsc_get_attr()
5707 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,  in kvm_arch_tsc_set_attr()  argument
5711 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_tsc_set_attr()
5726 		matched = (vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5727 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&  in kvm_arch_tsc_set_attr()
5730 		tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;  in kvm_arch_tsc_set_attr()
5734 		__kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);  in kvm_arch_tsc_set_attr()
5747 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_device_attr()  argument
5762 		r = kvm_arch_tsc_has_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5765 		r = kvm_arch_tsc_get_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5768 		r = kvm_arch_tsc_set_attr(vcpu, &attr);  in kvm_vcpu_ioctl_device_attr()
5775 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,  in kvm_vcpu_ioctl_enable_cap()  argument
5789 		if (!irqchip_in_kernel(vcpu->kvm))  in kvm_vcpu_ioctl_enable_cap()
5791 		return kvm_hv_activate_synic(vcpu, cap->cap ==  in kvm_vcpu_ioctl_enable_cap()
5801 			r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);  in kvm_vcpu_ioctl_enable_cap()
5814 		return kvm_x86_call(enable_l2_tlb_flush)(vcpu);  in kvm_vcpu_ioctl_enable_cap()
5817 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);  in kvm_vcpu_ioctl_enable_cap()
5821 		vcpu->arch.pv_cpuid.enforce = cap->args[0];  in kvm_vcpu_ioctl_enable_cap()
5831 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()  local
5842 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl()
5848 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5855 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5866 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5874 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);  in kvm_arch_vcpu_ioctl()
5883 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);  in kvm_arch_vcpu_ioctl()
5887 		r = kvm_vcpu_ioctl_nmi(vcpu);  in kvm_arch_vcpu_ioctl()
5891 		r = kvm_inject_smi(vcpu);  in kvm_arch_vcpu_ioctl()
5901 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);  in kvm_arch_vcpu_ioctl()
5911 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5922 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,  in kvm_arch_vcpu_ioctl()
5933 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5934 		r = msr_io(vcpu, argp, do_get_msr, 1);  in kvm_arch_vcpu_ioctl()
5935 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5939 		int idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5940 		r = msr_io(vcpu, argp, do_set_msr, 0);  in kvm_arch_vcpu_ioctl()
5941 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5950 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);  in kvm_arch_vcpu_ioctl()
5964 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl()
5969 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
5970 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);  in kvm_arch_vcpu_ioctl()
5971 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
5980 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);  in kvm_arch_vcpu_ioctl()
5989 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);  in kvm_arch_vcpu_ioctl()
5995 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
6010 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl()
6011 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);  in kvm_arch_vcpu_ioctl()
6012 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl()
6018 		r = kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
6037 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);  in kvm_arch_vcpu_ioctl()
6042 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))  in kvm_arch_vcpu_ioctl()
6050 		r = kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
6061 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
6069 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);  in kvm_arch_vcpu_ioctl()
6074 		int size = vcpu->arch.guest_fpu.uabi_size;  in kvm_arch_vcpu_ioctl()
6081 		r = kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);  in kvm_arch_vcpu_ioctl()
6099 		r = kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
6117 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);  in kvm_arch_vcpu_ioctl()
6133 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))  in kvm_arch_vcpu_ioctl()
6139 		r = vcpu->arch.virtual_tsc_khz;  in kvm_arch_vcpu_ioctl()
6143 		r = kvm_set_guest_paused(vcpu);  in kvm_arch_vcpu_ioctl()
6152 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);  in kvm_arch_vcpu_ioctl()
6168 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,  in kvm_arch_vcpu_ioctl()
6212 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl()
6213 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);  in kvm_arch_vcpu_ioctl()
6214 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl()
6219 		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);  in kvm_arch_vcpu_ioctl()
6229 		r = kvm_xen_vcpu_get_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6240 		r = kvm_xen_vcpu_set_attr(vcpu, &xva);  in kvm_arch_vcpu_ioctl()
6246 		if (vcpu->kvm->arch.has_protected_state &&  in kvm_arch_vcpu_ioctl()
6247 		    vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl()
6254 		__get_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6263 		if (vcpu->kvm->arch.has_protected_state &&  in kvm_arch_vcpu_ioctl()
6264 		    vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl()
6273 		r = __set_sregs2(vcpu, u.sregs2);  in kvm_arch_vcpu_ioctl()
6279 		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);  in kvm_arch_vcpu_ioctl()
6287 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl()
6291 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)  in kvm_arch_vcpu_fault()  argument
6465 	struct kvm_vcpu *vcpu;  in kvm_arch_sync_dirty_log()  local
6471 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_sync_dirty_log()
6472 		kvm_vcpu_kick(vcpu);  in kvm_arch_sync_dirty_log()
6911 	struct kvm_vcpu *vcpu;  in kvm_arch_suspend_notifier()  local
6916 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_suspend_notifier()
6917 		if (!vcpu->arch.pv_time.active)  in kvm_arch_suspend_notifier()
6920 		ret = kvm_set_guest_paused(vcpu);  in kvm_arch_suspend_notifier()
6922 			kvm_err("Failed to pause guest VCPU%d: %d\n",  in kvm_arch_suspend_notifier()
6923 				vcpu->vcpu_id, ret);  in kvm_arch_suspend_notifier()
7483 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,  in vcpu_mmio_write()  argument
7491 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_write()
7492 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))  in vcpu_mmio_write()
7493 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_write()
7504 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)  in vcpu_mmio_read()  argument
7511 		if (!(lapic_in_kernel(vcpu) &&  in vcpu_mmio_read()
7512 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,  in vcpu_mmio_read()
7514 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))  in vcpu_mmio_read()
7526 void kvm_set_segment(struct kvm_vcpu *vcpu,  in kvm_set_segment()  argument
7529 	kvm_x86_call(set_segment)(vcpu, var, seg);  in kvm_set_segment()
7532 void kvm_get_segment(struct kvm_vcpu *vcpu,  in kvm_get_segment()  argument
7535 	kvm_x86_call(get_segment)(vcpu, var, seg);  in kvm_get_segment()
7538 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,  in translate_nested_gpa()  argument
7541 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in translate_nested_gpa()
7544 	BUG_ON(!mmu_is_nested(vcpu));  in translate_nested_gpa()
7548 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);  in translate_nested_gpa()
7553 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_read()  argument
7556 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_read()
7558 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_read()
7559 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_read()
7563 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_write()  argument
7566 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_write()
7568 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_mmu_gva_to_gpa_write()
7570 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in kvm_mmu_gva_to_gpa_write()
7575 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,  in kvm_mmu_gva_to_gpa_system()  argument
7578 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_mmu_gva_to_gpa_system()
7580 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);  in kvm_mmu_gva_to_gpa_system()
7584 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_read_guest_virt_helper()  argument
7587 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_read_guest_virt_helper()
7592 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_read_guest_virt_helper()
7599 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,  in kvm_read_guest_virt_helper()
7619 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in kvm_fetch_guest_virt()  local
7620 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fetch_guest_virt()
7621 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_fetch_guest_virt()
7626 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,  in kvm_fetch_guest_virt()
7634 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,  in kvm_fetch_guest_virt()
7642 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,  in kvm_read_guest_virt()  argument
7646 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;  in kvm_read_guest_virt()
7655 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,  in kvm_read_guest_virt()
7664 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_std()  local
7669 	else if (kvm_x86_call(get_cpl)(vcpu) == 3)  in emulator_read_std()
7672 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);  in emulator_read_std()
7676 				      struct kvm_vcpu *vcpu, u64 access,  in kvm_write_guest_virt_helper()  argument
7679 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_write_guest_virt_helper()
7684 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);  in kvm_write_guest_virt_helper()
7691 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);  in kvm_write_guest_virt_helper()
7709 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_write_std()  local
7714 	else if (kvm_x86_call(get_cpl)(vcpu) == 3)  in emulator_write_std()
7717 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in emulator_write_std()
7721 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,  in kvm_write_guest_virt_system()  argument
7725 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_write_guest_virt_system()
7727 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,  in kvm_write_guest_virt_system()
7732 static int kvm_check_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,  in kvm_check_emulate_insn()  argument
7735 	return kvm_x86_call(check_emulate_instruction)(vcpu, emul_type,  in kvm_check_emulate_insn()
7739 int handle_ud(struct kvm_vcpu *vcpu)  in handle_ud()  argument
7748 	r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0);  in handle_ud()
7753 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),  in handle_ud()
7757 			kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);  in handle_ud()
7758 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));  in handle_ud()
7762 	return kvm_emulate_instruction(vcpu, emul_type);  in handle_ud()
7766 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_is_mmio_gpa()  argument
7773 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {  in vcpu_is_mmio_gpa()
7781 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,  in vcpu_mmio_gva_to_gpa()  argument
7785 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in vcpu_mmio_gva_to_gpa()
7786 	u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)  in vcpu_mmio_gva_to_gpa()
7794 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||  in vcpu_mmio_gva_to_gpa()
7795 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,  in vcpu_mmio_gva_to_gpa()
7796 			      vcpu->arch.mmio_access, 0, access))) {  in vcpu_mmio_gva_to_gpa()
7797 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |  in vcpu_mmio_gva_to_gpa()
7803 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);  in vcpu_mmio_gva_to_gpa()
7808 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);  in vcpu_mmio_gva_to_gpa()
7811 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,  in emulator_write_phys()  argument
7816 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);  in emulator_write_phys()
7819 	kvm_page_track_write(vcpu, gpa, val, bytes);  in emulator_write_phys()
7824 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7826 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7828 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7830 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7835 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)  in read_prepare()  argument
7837 	if (vcpu->mmio_read_completed) {  in read_prepare()
7839 			       vcpu->mmio_fragments[0].gpa, val);  in read_prepare()
7840 		vcpu->mmio_read_completed = 0;  in read_prepare()
7847 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_emulate()  argument
7850 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);  in read_emulate()
7853 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_emulate()  argument
7856 	return emulator_write_phys(vcpu, gpa, val, bytes);  in write_emulate()
7859 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)  in write_mmio()  argument
7862 	return vcpu_mmio_write(vcpu, gpa, bytes, val);  in write_mmio()
7865 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in read_exit_mmio()  argument
7872 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,  in write_exit_mmio()  argument
7875 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];  in write_exit_mmio()
7877 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in write_exit_mmio()
7898 				       struct kvm_vcpu *vcpu,  in emulator_read_write_onepage()  argument
7905 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in emulator_read_write_onepage()
7917 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);  in emulator_read_write_onepage()
7919 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);  in emulator_read_write_onepage()
7924 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))  in emulator_read_write_onepage()
7930 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);  in emulator_read_write_onepage()
7938 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);  in emulator_read_write_onepage()
7939 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];  in emulator_read_write_onepage()
7952 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_read_write()  local
7957 		  ops->read_write_prepare(vcpu, val, bytes))  in emulator_read_write()
7960 	vcpu->mmio_nr_fragments = 0;  in emulator_read_write()
7968 						 vcpu, ops);  in emulator_read_write()
7980 					 vcpu, ops);  in emulator_read_write()
7984 	if (!vcpu->mmio_nr_fragments)  in emulator_read_write()
7987 	gpa = vcpu->mmio_fragments[0].gpa;  in emulator_read_write()
7989 	vcpu->mmio_needed = 1;  in emulator_read_write()
7990 	vcpu->mmio_cur_fragment = 0;  in emulator_read_write()
7992 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);  in emulator_read_write()
7993 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;  in emulator_read_write()
7994 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in emulator_read_write()
7995 	vcpu->run->mmio.phys_addr = gpa;  in emulator_read_write()
7997 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);  in emulator_read_write()
8030 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_cmpxchg_emulated()  local
8040 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);  in emulator_cmpxchg_emulated()
8058 	hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
8091 	kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa));  in emulator_cmpxchg_emulated()
8096 	kvm_page_track_write(vcpu, gpa, new, bytes);  in emulator_cmpxchg_emulated()
8106 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in_out()  argument
8113 	WARN_ON_ONCE(vcpu->arch.pio.count);  in emulator_pio_in_out()
8116 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
8118 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);  in emulator_pio_in_out()
8138 	vcpu->arch.pio.port = port;  in emulator_pio_in_out()
8139 	vcpu->arch.pio.in = in;  in emulator_pio_in_out()
8140 	vcpu->arch.pio.count = count;  in emulator_pio_in_out()
8141 	vcpu->arch.pio.size = size;  in emulator_pio_in_out()
8144 		memset(vcpu->arch.pio_data, 0, size * count);  in emulator_pio_in_out()
8146 		memcpy(vcpu->arch.pio_data, data, size * count);  in emulator_pio_in_out()
8148 	vcpu->run->exit_reason = KVM_EXIT_IO;  in emulator_pio_in_out()
8149 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;  in emulator_pio_in_out()
8150 	vcpu->run->io.size = size;  in emulator_pio_in_out()
8151 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;  in emulator_pio_in_out()
8152 	vcpu->run->io.count = count;  in emulator_pio_in_out()
8153 	vcpu->run->io.port = port;  in emulator_pio_in_out()
8157 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,  in emulator_pio_in()  argument
8160 	int r = emulator_pio_in_out(vcpu, size, port, val, count, true);  in emulator_pio_in()
8167 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)  in complete_emulator_pio_in()  argument
8169 	int size = vcpu->arch.pio.size;  in complete_emulator_pio_in()
8170 	unsigned int count = vcpu->arch.pio.count;  in complete_emulator_pio_in()
8171 	memcpy(val, vcpu->arch.pio_data, size * count);  in complete_emulator_pio_in()
8172 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);  in complete_emulator_pio_in()
8173 	vcpu->arch.pio.count = 0;  in complete_emulator_pio_in()
8180 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_pio_in_emulated()  local
8181 	if (vcpu->arch.pio.count) {  in emulator_pio_in_emulated()
8185 		 * can modify ECX before rerunning the vCPU.  Ignore any such  in emulator_pio_in_emulated()
8189 		complete_emulator_pio_in(vcpu, val);  in emulator_pio_in_emulated()
8193 	return emulator_pio_in(vcpu, size, port, val, count);  in emulator_pio_in_emulated()
8196 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,  in emulator_pio_out()  argument
8201 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);  in emulator_pio_out()
8211 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)  in get_segment_base()  argument
8213 	return kvm_x86_call(get_segment_base)(vcpu, seg);  in get_segment_base()
8221 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd_noskip()  argument
8223 	if (!need_emulate_wbinvd(vcpu))  in kvm_emulate_wbinvd_noskip()
8229 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
8230 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,  in kvm_emulate_wbinvd_noskip()
8233 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);  in kvm_emulate_wbinvd_noskip()
8239 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)  in kvm_emulate_wbinvd()  argument
8241 	kvm_emulate_wbinvd_noskip(vcpu);  in kvm_emulate_wbinvd()
8242 	return kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_wbinvd()
8272 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_cr()  local
8277 		value = kvm_read_cr0(vcpu);  in emulator_get_cr()
8280 		value = vcpu->arch.cr2;  in emulator_get_cr()
8283 		value = kvm_read_cr3(vcpu);  in emulator_get_cr()
8286 		value = kvm_read_cr4(vcpu);  in emulator_get_cr()
8289 		value = kvm_get_cr8(vcpu);  in emulator_get_cr()
8301 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_cr()  local
8306 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));  in emulator_set_cr()
8309 		vcpu->arch.cr2 = val;  in emulator_set_cr()
8312 		res = kvm_set_cr3(vcpu, val);  in emulator_set_cr()
8315 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));  in emulator_set_cr()
8318 		res = kvm_set_cr8(vcpu, val);  in emulator_set_cr()
8399 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_segment()  local
8421 	kvm_set_segment(vcpu, &var, seg);  in emulator_set_segment()
8428 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_get_msr_with_filter()  local
8431 	r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);  in emulator_get_msr_with_filter()
8436 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,  in emulator_get_msr_with_filter()
8451 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_set_msr_with_filter()  local
8454 	r = kvm_set_msr_with_filter(vcpu, msr_index, data);  in emulator_set_msr_with_filter()
8459 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,  in emulator_set_msr_with_filter()
8644 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)  in toggle_interruptibility()  argument
8646 	u32 int_shadow = kvm_x86_call(get_interrupt_shadow)(vcpu);  in toggle_interruptibility()
8657 		kvm_x86_call(set_interrupt_shadow)(vcpu, mask);  in toggle_interruptibility()
8659 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in toggle_interruptibility()
8663 static void inject_emulated_exception(struct kvm_vcpu *vcpu)  in inject_emulated_exception()  argument
8665 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in inject_emulated_exception()
8668 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);  in inject_emulated_exception()
8670 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,  in inject_emulated_exception()
8673 		kvm_queue_exception(vcpu, ctxt->exception.vector);  in inject_emulated_exception()
8676 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)  in alloc_emulate_ctxt()  argument
8682 		pr_err("failed to allocate vcpu's emulator\n");  in alloc_emulate_ctxt()
8686 	ctxt->vcpu = vcpu;  in alloc_emulate_ctxt()
8688 	vcpu->arch.emulate_ctxt = ctxt;  in alloc_emulate_ctxt()
8693 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)  in init_emulate_ctxt()  argument
8695 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in init_emulate_ctxt()
8698 	kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);  in init_emulate_ctxt()
8701 	ctxt->eflags = kvm_get_rflags(vcpu);  in init_emulate_ctxt()
8704 	ctxt->eip = kvm_rip_read(vcpu);  in init_emulate_ctxt()
8705 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :  in init_emulate_ctxt()
8707 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :  in init_emulate_ctxt()
8716 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in init_emulate_ctxt()
8719 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)  in kvm_inject_realmode_interrupt()  argument
8721 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_inject_realmode_interrupt()
8724 	init_emulate_ctxt(vcpu);  in kvm_inject_realmode_interrupt()
8732 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);  in kvm_inject_realmode_interrupt()
8735 		kvm_rip_write(vcpu, ctxt->eip);  in kvm_inject_realmode_interrupt()
8736 		kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_inject_realmode_interrupt()
8741 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in prepare_emulation_failure_exit()  argument
8744 	struct kvm_run *run = vcpu->run;  in prepare_emulation_failure_exit()
8754 	kvm_x86_call(get_exit_info)(vcpu, (u32 *)&info[0], &info[1], &info[2],  in prepare_emulation_failure_exit()
8791 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)  in prepare_emulation_ctxt_failure_exit()  argument
8793 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in prepare_emulation_ctxt_failure_exit()
8795 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,  in prepare_emulation_ctxt_failure_exit()
8799 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,  in __kvm_prepare_emulation_failure_exit()  argument
8802 	prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);  in __kvm_prepare_emulation_failure_exit()
8806 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)  in kvm_prepare_emulation_failure_exit()  argument
8808 	__kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);  in kvm_prepare_emulation_failure_exit()
8812 void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa)  in kvm_prepare_event_vectoring_exit()  argument
8815 	struct kvm_run *run = vcpu->run;  in kvm_prepare_event_vectoring_exit()
8819 	kvm_x86_call(get_exit_info)(vcpu, &reason, &info1, &info2,  in kvm_prepare_event_vectoring_exit()
8826 	run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu;  in kvm_prepare_event_vectoring_exit()
8834 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)  in handle_emulation_failure()  argument
8836 	struct kvm *kvm = vcpu->kvm;  in handle_emulation_failure()
8838 	++vcpu->stat.insn_emulation_fail;  in handle_emulation_failure()
8839 	trace_kvm_emulate_insn_failed(vcpu);  in handle_emulation_failure()
8842 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in handle_emulation_failure()
8848 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8852 	kvm_queue_exception(vcpu, UD_VECTOR);  in handle_emulation_failure()
8854 	if (!is_guest_mode(vcpu) && kvm_x86_call(get_cpl)(vcpu) == 0) {  in handle_emulation_failure()
8855 		prepare_emulation_ctxt_failure_exit(vcpu);  in handle_emulation_failure()
8862 static bool kvm_unprotect_and_retry_on_failure(struct kvm_vcpu *vcpu,  in kvm_unprotect_and_retry_on_failure()  argument
8874 	 * the vCPU into an infinite loop of page faults.  E.g. KVM will create  in kvm_unprotect_and_retry_on_failure()
8888 	__kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, true);  in kvm_unprotect_and_retry_on_failure()
8891 	 * Retry even if _this_ vCPU didn't unprotect the gfn, as it's possible  in kvm_unprotect_and_retry_on_failure()
8900 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8901 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8918 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)  in kvm_vcpu_do_singlestep()  argument
8920 	struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_do_singlestep()
8922 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {  in kvm_vcpu_do_singlestep()
8924 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);  in kvm_vcpu_do_singlestep()
8929 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);  in kvm_vcpu_do_singlestep()
8933 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)  in kvm_skip_emulated_instruction()  argument
8935 	unsigned long rflags = kvm_x86_call(get_rflags)(vcpu);  in kvm_skip_emulated_instruction()
8938 	r = kvm_x86_call(skip_emulated_instruction)(vcpu);  in kvm_skip_emulated_instruction()
8942 	kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED);  in kvm_skip_emulated_instruction()
8953 		r = kvm_vcpu_do_singlestep(vcpu);  in kvm_skip_emulated_instruction()
8958 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)  in kvm_is_code_breakpoint_inhibited()  argument
8960 	if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)  in kvm_is_code_breakpoint_inhibited()
8967 	if (!guest_cpuid_is_intel_compatible(vcpu))  in kvm_is_code_breakpoint_inhibited()
8970 	return kvm_x86_call(get_interrupt_shadow)(vcpu) & KVM_X86_SHADOW_INT_MOV_SS;  in kvm_is_code_breakpoint_inhibited()
8973 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,  in kvm_vcpu_check_code_breakpoint()  argument
8996 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&  in kvm_vcpu_check_code_breakpoint()
8997 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {  in kvm_vcpu_check_code_breakpoint()
8998 		struct kvm_run *kvm_run = vcpu->run;  in kvm_vcpu_check_code_breakpoint()
8999 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
9001 					   vcpu->arch.guest_debug_dr7,  in kvm_vcpu_check_code_breakpoint()
9002 					   vcpu->arch.eff_db);  in kvm_vcpu_check_code_breakpoint()
9014 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&  in kvm_vcpu_check_code_breakpoint()
9015 	    !kvm_is_code_breakpoint_inhibited(vcpu)) {  in kvm_vcpu_check_code_breakpoint()
9016 		unsigned long eip = kvm_get_linear_rip(vcpu);  in kvm_vcpu_check_code_breakpoint()
9018 					   vcpu->arch.dr7,  in kvm_vcpu_check_code_breakpoint()
9019 					   vcpu->arch.db);  in kvm_vcpu_check_code_breakpoint()
9022 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);  in kvm_vcpu_check_code_breakpoint()
9072 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,  in x86_decode_emulated_instruction()  argument
9075 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_decode_emulated_instruction()
9078 	init_emulate_ctxt(vcpu);  in x86_decode_emulated_instruction()
9082 	trace_kvm_emulate_insn_start(vcpu);  in x86_decode_emulated_instruction()
9083 	++vcpu->stat.insn_emulation;  in x86_decode_emulated_instruction()
9089 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,  in x86_emulate_instruction()  argument
9093 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in x86_emulate_instruction()
9097 	    (WARN_ON_ONCE(is_guest_mode(vcpu)) ||  in x86_emulate_instruction()
9101 	r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len);  in x86_emulate_instruction()
9106 		if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
9111 			kvm_prepare_event_vectoring_exit(vcpu, cr2_or_gpa);  in x86_emulate_instruction()
9116 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9119 	vcpu->arch.l1tf_flush_l1d = true;  in x86_emulate_instruction()
9122 		kvm_clear_exception_queue(vcpu);  in x86_emulate_instruction()
9129 		if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))  in x86_emulate_instruction()
9132 		r = x86_decode_emulated_instruction(vcpu, emulation_type,  in x86_emulate_instruction()
9137 				kvm_queue_exception(vcpu, UD_VECTOR);  in x86_emulate_instruction()
9140 			if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
9152 				inject_emulated_exception(vcpu);  in x86_emulate_instruction()
9155 			return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9161 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);  in x86_emulate_instruction()
9182 		kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9184 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);  in x86_emulate_instruction()
9191 	 * and retry the instruction, as the vCPU is likely no longer using the  in x86_emulate_instruction()
9196 	    kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa))  in x86_emulate_instruction()
9201 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {  in x86_emulate_instruction()
9202 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;  in x86_emulate_instruction()
9212 		if (vcpu->arch.mmu->root_role.direct) {  in x86_emulate_instruction()
9227 		if (kvm_unprotect_and_retry_on_failure(vcpu, cr2_or_gpa,  in x86_emulate_instruction()
9231 		return handle_emulation_failure(vcpu, emulation_type);  in x86_emulate_instruction()
9235 		WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);  in x86_emulate_instruction()
9236 		vcpu->mmio_needed = false;  in x86_emulate_instruction()
9238 		inject_emulated_exception(vcpu);  in x86_emulate_instruction()
9239 	} else if (vcpu->arch.pio.count) {  in x86_emulate_instruction()
9240 		if (!vcpu->arch.pio.in) {  in x86_emulate_instruction()
9242 			vcpu->arch.pio.count = 0;  in x86_emulate_instruction()
9245 			vcpu->arch.complete_userspace_io = complete_emulated_pio;  in x86_emulate_instruction()
9248 	} else if (vcpu->mmio_needed) {  in x86_emulate_instruction()
9249 		++vcpu->stat.mmio_exits;  in x86_emulate_instruction()
9251 		if (!vcpu->mmio_is_write)  in x86_emulate_instruction()
9254 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in x86_emulate_instruction()
9255 	} else if (vcpu->arch.complete_userspace_io) {  in x86_emulate_instruction()
9265 		unsigned long rflags = kvm_x86_call(get_rflags)(vcpu);  in x86_emulate_instruction()
9266 		toggle_interruptibility(vcpu, ctxt->interruptibility);  in x86_emulate_instruction()
9267 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in x86_emulate_instruction()
9276 			kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED);  in x86_emulate_instruction()
9278 				kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED);  in x86_emulate_instruction()
9279 			kvm_rip_write(vcpu, ctxt->eip);  in x86_emulate_instruction()
9280 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))  in x86_emulate_instruction()
9281 				r = kvm_vcpu_do_singlestep(vcpu);  in x86_emulate_instruction()
9282 			kvm_x86_call(update_emulated_instruction)(vcpu);  in x86_emulate_instruction()
9283 			__kvm_set_rflags(vcpu, ctxt->eflags);  in x86_emulate_instruction()
9293 			kvm_make_request(KVM_REQ_EVENT, vcpu);  in x86_emulate_instruction()
9295 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;  in x86_emulate_instruction()
9300 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)  in kvm_emulate_instruction()  argument
9302 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);  in kvm_emulate_instruction()
9306 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,  in kvm_emulate_instruction_from_buffer()  argument
9309 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);  in kvm_emulate_instruction_from_buffer()
9313 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)  in complete_fast_pio_out_port_0x7e()  argument
9315 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out_port_0x7e()
9319 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)  in complete_fast_pio_out()  argument
9321 	vcpu->arch.pio.count = 0;  in complete_fast_pio_out()
9323 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))  in complete_fast_pio_out()
9326 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_out()
9329 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_out()  argument
9332 	unsigned long val = kvm_rax_read(vcpu);  in kvm_fast_pio_out()
9333 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);  in kvm_fast_pio_out()
9343 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {  in kvm_fast_pio_out()
9344 		vcpu->arch.complete_userspace_io =  in kvm_fast_pio_out()
9346 		kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio_out()
9348 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_out()
9349 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;  in kvm_fast_pio_out()
9354 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)  in complete_fast_pio_in()  argument
9359 	BUG_ON(vcpu->arch.pio.count != 1);  in complete_fast_pio_in()
9361 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {  in complete_fast_pio_in()
9362 		vcpu->arch.pio.count = 0;  in complete_fast_pio_in()
9367 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;  in complete_fast_pio_in()
9369 	complete_emulator_pio_in(vcpu, &val);  in complete_fast_pio_in()
9370 	kvm_rax_write(vcpu, val);  in complete_fast_pio_in()
9372 	return kvm_skip_emulated_instruction(vcpu);  in complete_fast_pio_in()
9375 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,  in kvm_fast_pio_in()  argument
9382 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;  in kvm_fast_pio_in()
9384 	ret = emulator_pio_in(vcpu, size, port, &val, 1);  in kvm_fast_pio_in()
9386 		kvm_rax_write(vcpu, val);  in kvm_fast_pio_in()
9390 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);  in kvm_fast_pio_in()
9391 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;  in kvm_fast_pio_in()
9396 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)  in kvm_fast_pio()  argument
9401 		ret = kvm_fast_pio_in(vcpu, size, port);  in kvm_fast_pio()
9403 		ret = kvm_fast_pio_out(vcpu, size, port);  in kvm_fast_pio()
9404 	return ret && kvm_skip_emulated_instruction(vcpu);  in kvm_fast_pio()
9463 	struct kvm_vcpu *vcpu;  in __kvmclock_cpufreq_notifier()  local
9478 	 * the TSC for each VCPU.  We must flag these local variables  in __kvmclock_cpufreq_notifier()
9497 	 * anytime after the setting of the VCPU's request bit, the  in __kvmclock_cpufreq_notifier()
9510 		kvm_for_each_vcpu(i, vcpu, kvm) {  in __kvmclock_cpufreq_notifier()
9511 			if (vcpu->cpu != cpu)  in __kvmclock_cpufreq_notifier()
9513 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in __kvmclock_cpufreq_notifier()
9514 			if (vcpu->cpu != raw_smp_processor_id())  in __kvmclock_cpufreq_notifier()
9594 	struct kvm_vcpu *vcpu;  in pvclock_gtod_update_fn()  local
9599 		kvm_for_each_vcpu(i, vcpu, kvm)  in pvclock_gtod_update_fn()
9600 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in pvclock_gtod_update_fn()
9704 	 * vCPU's FPU state as a fxregs_state struct.  in kvm_x86_vendor_init()
9866 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,  in kvm_pv_clock_pairing()  argument
9881 	if (vcpu->arch.tsc_always_catchup)  in kvm_pv_clock_pairing()
9889 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);  in kvm_pv_clock_pairing()
9894 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,  in kvm_pv_clock_pairing()
9903  * kvm_pv_kick_cpu_op:  Kick a vcpu.
9905  * @apicid - apicid of vcpu to be kicked.
9929 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)  in kvm_vcpu_apicv_activated()  argument
9931 	ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);  in kvm_vcpu_apicv_activated()
9933 			kvm_x86_call(vcpu_get_apicv_inhibit_reasons)(vcpu);  in kvm_vcpu_apicv_activated()
9964 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)  in kvm_sched_yield()  argument
9969 	vcpu->stat.directed_yield_attempted++;  in kvm_sched_yield()
9975 	map = rcu_dereference(vcpu->kvm->arch.apic_map);  in kvm_sched_yield()
9978 		target = map->phys_map[dest_id]->vcpu;  in kvm_sched_yield()
9986 	if (vcpu == target)  in kvm_sched_yield()
9992 	vcpu->stat.directed_yield_successful++;  in kvm_sched_yield()
9998 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)  in complete_hypercall_exit()  argument
10000 	u64 ret = vcpu->run->hypercall.ret;  in complete_hypercall_exit()
10002 	if (!is_64_bit_hypercall(vcpu))  in complete_hypercall_exit()
10004 	kvm_rax_write(vcpu, ret);  in complete_hypercall_exit()
10005 	return kvm_skip_emulated_instruction(vcpu);  in complete_hypercall_exit()
10008 int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, unsigned long nr,  in ____kvm_emulate_hypercall()  argument
10016 	++vcpu->stat.hypercalls;  in ____kvm_emulate_hypercall()
10040 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))  in ____kvm_emulate_hypercall()
10043 		kvm_pv_kick_cpu_op(vcpu->kvm, a1);  in ____kvm_emulate_hypercall()
10044 		kvm_sched_yield(vcpu, a1);  in ____kvm_emulate_hypercall()
10049 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);  in ____kvm_emulate_hypercall()
10053 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))  in ____kvm_emulate_hypercall()
10056 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);  in ____kvm_emulate_hypercall()
10059 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))  in ____kvm_emulate_hypercall()
10062 		kvm_sched_yield(vcpu, a0);  in ____kvm_emulate_hypercall()
10069 		if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE))  in ____kvm_emulate_hypercall()
10078 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;  in ____kvm_emulate_hypercall()
10079 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;  in ____kvm_emulate_hypercall()
10082 		 * assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that  in ____kvm_emulate_hypercall()
10084 		 * vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU.  in ____kvm_emulate_hypercall()
10086 		vcpu->run->hypercall.ret = 0;  in ____kvm_emulate_hypercall()
10087 		vcpu->run->hypercall.args[0]  = gpa;  in ____kvm_emulate_hypercall()
10088 		vcpu->run->hypercall.args[1]  = npages;  in ____kvm_emulate_hypercall()
10089 		vcpu->run->hypercall.args[2]  = attrs;  in ____kvm_emulate_hypercall()
10090 		vcpu->run->hypercall.flags    = 0;  in ____kvm_emulate_hypercall()
10092 			vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;  in ____kvm_emulate_hypercall()
10094 		WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);  in ____kvm_emulate_hypercall()
10095 		vcpu->arch.complete_userspace_io = complete_hypercall;  in ____kvm_emulate_hypercall()
10104 	vcpu->run->hypercall.ret = ret;  in ____kvm_emulate_hypercall()
10109 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)  in kvm_emulate_hypercall()  argument
10111 	if (kvm_xen_hypercall_enabled(vcpu->kvm))  in kvm_emulate_hypercall()
10112 		return kvm_xen_hypercall(vcpu);  in kvm_emulate_hypercall()
10114 	if (kvm_hv_hypercall_enabled(vcpu))  in kvm_emulate_hypercall()
10115 		return kvm_hv_hypercall(vcpu);  in kvm_emulate_hypercall()
10117 	return __kvm_emulate_hypercall(vcpu, rax, rbx, rcx, rdx, rsi,  in kvm_emulate_hypercall()
10118 				       is_64_bit_hypercall(vcpu),  in kvm_emulate_hypercall()
10119 				       kvm_x86_call(get_cpl)(vcpu),  in kvm_emulate_hypercall()
10126 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);  in emulator_fix_hypercall()  local
10128 	unsigned long rip = kvm_rip_read(vcpu);  in emulator_fix_hypercall()
10134 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {  in emulator_fix_hypercall()
10141 	kvm_x86_call(patch_hypercall)(vcpu, instruction);  in emulator_fix_hypercall()
10147 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)  in dm_request_for_irq_injection()  argument
10149 	return vcpu->run->request_interrupt_window &&  in dm_request_for_irq_injection()
10150 		likely(!pic_in_kernel(vcpu->kvm));  in dm_request_for_irq_injection()
10154 static void post_kvm_run_save(struct kvm_vcpu *vcpu)  in post_kvm_run_save()  argument
10156 	struct kvm_run *kvm_run = vcpu->run;  in post_kvm_run_save()
10158 	kvm_run->if_flag = kvm_x86_call(get_if_flag)(vcpu);  in post_kvm_run_save()
10159 	kvm_run->cr8 = kvm_get_cr8(vcpu);  in post_kvm_run_save()
10160 	kvm_run->apic_base = vcpu->arch.apic_base;  in post_kvm_run_save()
10163 		pic_in_kernel(vcpu->kvm) ||  in post_kvm_run_save()
10164 		kvm_vcpu_ready_for_interrupt_injection(vcpu);  in post_kvm_run_save()
10166 	if (is_smm(vcpu))  in post_kvm_run_save()
10168 	if (is_guest_mode(vcpu))  in post_kvm_run_save()
10172 static void update_cr8_intercept(struct kvm_vcpu *vcpu)  in update_cr8_intercept()  argument
10179 	if (!lapic_in_kernel(vcpu))  in update_cr8_intercept()
10182 	if (vcpu->arch.apic->apicv_active)  in update_cr8_intercept()
10185 	if (!vcpu->arch.apic->vapic_addr)  in update_cr8_intercept()
10186 		max_irr = kvm_lapic_find_highest_irr(vcpu);  in update_cr8_intercept()
10193 	tpr = kvm_lapic_get_cr8(vcpu);  in update_cr8_intercept()
10195 	kvm_x86_call(update_cr8_intercept)(vcpu, tpr, max_irr);  in update_cr8_intercept()
10199 int kvm_check_nested_events(struct kvm_vcpu *vcpu)  in kvm_check_nested_events()  argument
10201 	if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in kvm_check_nested_events()
10202 		kvm_x86_ops.nested_ops->triple_fault(vcpu);  in kvm_check_nested_events()
10206 	return kvm_x86_ops.nested_ops->check_events(vcpu);  in kvm_check_nested_events()
10209 static void kvm_inject_exception(struct kvm_vcpu *vcpu)  in kvm_inject_exception()  argument
10212 	 * Suppress the error code if the vCPU is in Real Mode, as Real Mode  in kvm_inject_exception()
10218 	vcpu->arch.exception.has_error_code &= is_protmode(vcpu);  in kvm_inject_exception()
10220 	trace_kvm_inj_exception(vcpu->arch.exception.vector,  in kvm_inject_exception()
10221 				vcpu->arch.exception.has_error_code,  in kvm_inject_exception()
10222 				vcpu->arch.exception.error_code,  in kvm_inject_exception()
10223 				vcpu->arch.exception.injected);  in kvm_inject_exception()
10225 	kvm_x86_call(inject_exception)(vcpu);  in kvm_inject_exception()
10267 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,  in kvm_check_and_inject_events()  argument
10278 	if (is_guest_mode(vcpu))  in kvm_check_and_inject_events()
10279 		r = kvm_check_nested_events(vcpu);  in kvm_check_and_inject_events()
10306 	if (vcpu->arch.exception.injected)  in kvm_check_and_inject_events()
10307 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10308 	else if (kvm_is_exception_pending(vcpu))  in kvm_check_and_inject_events()
10310 	else if (vcpu->arch.nmi_injected)  in kvm_check_and_inject_events()
10311 		kvm_x86_call(inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10312 	else if (vcpu->arch.interrupt.injected)  in kvm_check_and_inject_events()
10313 		kvm_x86_call(inject_irq)(vcpu, true);  in kvm_check_and_inject_events()
10320 	WARN_ON_ONCE(vcpu->arch.exception.injected &&  in kvm_check_and_inject_events()
10321 		     vcpu->arch.exception.pending);  in kvm_check_and_inject_events()
10340 	WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||  in kvm_check_and_inject_events()
10341 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10348 	can_inject = !kvm_event_needs_reinjection(vcpu);  in kvm_check_and_inject_events()
10350 	if (vcpu->arch.exception.pending) {  in kvm_check_and_inject_events()
10361 		if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)  in kvm_check_and_inject_events()
10362 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |  in kvm_check_and_inject_events()
10365 		if (vcpu->arch.exception.vector == DB_VECTOR) {  in kvm_check_and_inject_events()
10366 			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);  in kvm_check_and_inject_events()
10367 			if (vcpu->arch.dr7 & DR7_GD) {  in kvm_check_and_inject_events()
10368 				vcpu->arch.dr7 &= ~DR7_GD;  in kvm_check_and_inject_events()
10369 				kvm_update_dr7(vcpu);  in kvm_check_and_inject_events()
10373 		kvm_inject_exception(vcpu);  in kvm_check_and_inject_events()
10375 		vcpu->arch.exception.pending = false;  in kvm_check_and_inject_events()
10376 		vcpu->arch.exception.injected = true;  in kvm_check_and_inject_events()
10382 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)  in kvm_check_and_inject_events()
10397 	if (vcpu->arch.smi_pending) {  in kvm_check_and_inject_events()
10398 		r = can_inject ? kvm_x86_call(smi_allowed)(vcpu, true) :  in kvm_check_and_inject_events()
10403 			vcpu->arch.smi_pending = false;  in kvm_check_and_inject_events()
10404 			++vcpu->arch.smi_count;  in kvm_check_and_inject_events()
10405 			enter_smm(vcpu);  in kvm_check_and_inject_events()
10408 			kvm_x86_call(enable_smi_window)(vcpu);  in kvm_check_and_inject_events()
10412 	if (vcpu->arch.nmi_pending) {  in kvm_check_and_inject_events()
10413 		r = can_inject ? kvm_x86_call(nmi_allowed)(vcpu, true) :  in kvm_check_and_inject_events()
10418 			--vcpu->arch.nmi_pending;  in kvm_check_and_inject_events()
10419 			vcpu->arch.nmi_injected = true;  in kvm_check_and_inject_events()
10420 			kvm_x86_call(inject_nmi)(vcpu);  in kvm_check_and_inject_events()
10422 			WARN_ON(kvm_x86_call(nmi_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10424 		if (vcpu->arch.nmi_pending)  in kvm_check_and_inject_events()
10425 			kvm_x86_call(enable_nmi_window)(vcpu);  in kvm_check_and_inject_events()
10428 	if (kvm_cpu_has_injectable_intr(vcpu)) {  in kvm_check_and_inject_events()
10429 		r = can_inject ? kvm_x86_call(interrupt_allowed)(vcpu, true) :  in kvm_check_and_inject_events()
10434 			int irq = kvm_cpu_get_interrupt(vcpu);  in kvm_check_and_inject_events()
10437 				kvm_queue_interrupt(vcpu, irq, false);  in kvm_check_and_inject_events()
10438 				kvm_x86_call(inject_irq)(vcpu, false);  in kvm_check_and_inject_events()
10439 				WARN_ON(kvm_x86_call(interrupt_allowed)(vcpu, true) < 0);  in kvm_check_and_inject_events()
10442 		if (kvm_cpu_has_injectable_intr(vcpu))  in kvm_check_and_inject_events()
10443 			kvm_x86_call(enable_irq_window)(vcpu);  in kvm_check_and_inject_events()
10446 	if (is_guest_mode(vcpu) &&  in kvm_check_and_inject_events()
10448 	    kvm_x86_ops.nested_ops->has_events(vcpu, true))  in kvm_check_and_inject_events()
10454 	 * to the VMCS/VMCB.  Queueing a new exception can put the vCPU into an  in kvm_check_and_inject_events()
10459 	 * vCPU into an infinite loop.  Triple fault can be queued when running  in kvm_check_and_inject_events()
10463 	WARN_ON_ONCE(vcpu->arch.exception.pending ||  in kvm_check_and_inject_events()
10464 		     vcpu->arch.exception_vmexit.pending);  in kvm_check_and_inject_events()
10475 static void process_nmi(struct kvm_vcpu *vcpu)  in process_nmi()  argument
10481 	 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is  in process_nmi()
10489 	if (kvm_x86_call(get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)  in process_nmi()
10496 	 * tracked in vcpu->arch.nmi_pending.  in process_nmi()
10498 	if (kvm_x86_call(is_vnmi_pending)(vcpu))  in process_nmi()
10501 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);  in process_nmi()
10502 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);  in process_nmi()
10504 	if (vcpu->arch.nmi_pending &&  in process_nmi()
10505 	    (kvm_x86_call(set_vnmi_pending)(vcpu)))  in process_nmi()
10506 		vcpu->arch.nmi_pending--;  in process_nmi()
10508 	if (vcpu->arch.nmi_pending)  in process_nmi()
10509 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in process_nmi()
10513 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu)  in kvm_get_nr_pending_nmis()  argument
10515 	return vcpu->arch.nmi_pending +  in kvm_get_nr_pending_nmis()
10516 	       kvm_x86_call(is_vnmi_pending)(vcpu);  in kvm_get_nr_pending_nmis()
10530 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in __kvm_vcpu_update_apicv()  argument
10532 	struct kvm_lapic *apic = vcpu->arch.apic;  in __kvm_vcpu_update_apicv()
10535 	if (!lapic_in_kernel(vcpu))  in __kvm_vcpu_update_apicv()
10538 	down_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10542 	activate = kvm_vcpu_apicv_activated(vcpu) &&  in __kvm_vcpu_update_apicv()
10543 		   (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);  in __kvm_vcpu_update_apicv()
10549 	kvm_apic_update_apicv(vcpu);  in __kvm_vcpu_update_apicv()
10550 	kvm_x86_call(refresh_apicv_exec_ctrl)(vcpu);  in __kvm_vcpu_update_apicv()
10559 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __kvm_vcpu_update_apicv()
10563 	up_read(&vcpu->kvm->arch.apicv_update_lock);  in __kvm_vcpu_update_apicv()
10567 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)  in kvm_vcpu_update_apicv()  argument
10569 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_update_apicv()
10574 	 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but  in kvm_vcpu_update_apicv()
10579 	 * the vCPU would incorrectly be able to access the vAPIC page via MMIO  in kvm_vcpu_update_apicv()
10583 	if (apic_x2apic_mode(vcpu->arch.apic) &&  in kvm_vcpu_update_apicv()
10585 		kvm_inhibit_apic_access_page(vcpu);  in kvm_vcpu_update_apicv()
10587 	__kvm_vcpu_update_apicv(vcpu);  in kvm_vcpu_update_apicv()
10643 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)  in vcpu_scan_ioapic()  argument
10645 	if (!kvm_apic_present(vcpu))  in vcpu_scan_ioapic()
10648 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);  in vcpu_scan_ioapic()
10650 	kvm_x86_call(sync_pir_to_irr)(vcpu);  in vcpu_scan_ioapic()
10652 	if (irqchip_split(vcpu->kvm))  in vcpu_scan_ioapic()
10653 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10654 	else if (ioapic_in_kernel(vcpu->kvm))  in vcpu_scan_ioapic()
10655 		kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);  in vcpu_scan_ioapic()
10657 	if (is_guest_mode(vcpu))  in vcpu_scan_ioapic()
10658 		vcpu->arch.load_eoi_exitmap_pending = true;  in vcpu_scan_ioapic()
10660 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);  in vcpu_scan_ioapic()
10663 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)  in vcpu_load_eoi_exitmap()  argument
10665 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))  in vcpu_load_eoi_exitmap()
10669 	if (to_hv_vcpu(vcpu)) {  in vcpu_load_eoi_exitmap()
10673 			  vcpu->arch.ioapic_handled_vectors,  in vcpu_load_eoi_exitmap()
10674 			  to_hv_synic(vcpu)->vec_bitmap, 256);  in vcpu_load_eoi_exitmap()
10675 		kvm_x86_call(load_eoi_exitmap)(vcpu, eoi_exit_bitmap);  in vcpu_load_eoi_exitmap()
10680 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);  in vcpu_load_eoi_exitmap()
10688 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)  in kvm_vcpu_reload_apic_access_page()  argument
10690 	if (!lapic_in_kernel(vcpu))  in kvm_vcpu_reload_apic_access_page()
10693 	kvm_x86_call(set_apic_access_page_addr)(vcpu);  in kvm_vcpu_reload_apic_access_page()
10702 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)  in vcpu_enter_guest()  argument
10706 		dm_request_for_irq_injection(vcpu) &&  in vcpu_enter_guest()
10707 		kvm_cpu_accept_dm_intr(vcpu);  in vcpu_enter_guest()
10712 	if (kvm_request_pending(vcpu)) {  in vcpu_enter_guest()
10713 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {  in vcpu_enter_guest()
10718 		if (kvm_dirty_ring_check_request(vcpu)) {  in vcpu_enter_guest()
10723 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {  in vcpu_enter_guest()
10724 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {  in vcpu_enter_guest()
10729 		if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))  in vcpu_enter_guest()
10730 			kvm_mmu_free_obsolete_roots(vcpu);  in vcpu_enter_guest()
10731 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))  in vcpu_enter_guest()
10732 			__kvm_migrate_timers(vcpu);  in vcpu_enter_guest()
10733 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10734 			kvm_update_masterclock(vcpu->kvm);  in vcpu_enter_guest()
10735 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))  in vcpu_enter_guest()
10736 			kvm_gen_kvmclock_update(vcpu);  in vcpu_enter_guest()
10737 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {  in vcpu_enter_guest()
10738 			r = kvm_guest_time_update(vcpu);  in vcpu_enter_guest()
10742 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))  in vcpu_enter_guest()
10743 			kvm_mmu_sync_roots(vcpu);  in vcpu_enter_guest()
10744 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))  in vcpu_enter_guest()
10745 			kvm_mmu_load_pgd(vcpu);  in vcpu_enter_guest()
10752 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))  in vcpu_enter_guest()
10753 			kvm_vcpu_flush_tlb_all(vcpu);  in vcpu_enter_guest()
10755 		kvm_service_local_tlb_flush_requests(vcpu);  in vcpu_enter_guest()
10759 		 * flushing fails.  Note, Hyper-V's flushing is per-vCPU, but  in vcpu_enter_guest()
10764 		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) &&  in vcpu_enter_guest()
10765 		    kvm_hv_vcpu_flush_tlb(vcpu))  in vcpu_enter_guest()
10766 			kvm_vcpu_flush_tlb_guest(vcpu);  in vcpu_enter_guest()
10769 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {  in vcpu_enter_guest()
10770 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;  in vcpu_enter_guest()
10774 		if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10775 			if (is_guest_mode(vcpu))  in vcpu_enter_guest()
10776 				kvm_x86_ops.nested_ops->triple_fault(vcpu);  in vcpu_enter_guest()
10778 			if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {  in vcpu_enter_guest()
10779 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;  in vcpu_enter_guest()
10780 				vcpu->mmio_needed = 0;  in vcpu_enter_guest()
10785 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {  in vcpu_enter_guest()
10787 			vcpu->arch.apf.halted = true;  in vcpu_enter_guest()
10791 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))  in vcpu_enter_guest()
10792 			record_steal_time(vcpu);  in vcpu_enter_guest()
10793 		if (kvm_check_request(KVM_REQ_PMU, vcpu))  in vcpu_enter_guest()
10794 			kvm_pmu_handle_event(vcpu);  in vcpu_enter_guest()
10795 		if (kvm_check_request(KVM_REQ_PMI, vcpu))  in vcpu_enter_guest()
10796 			kvm_pmu_deliver_pmi(vcpu);  in vcpu_enter_guest()
10798 		if (kvm_check_request(KVM_REQ_SMI, vcpu))  in vcpu_enter_guest()
10799 			process_smi(vcpu);  in vcpu_enter_guest()
10801 		if (kvm_check_request(KVM_REQ_NMI, vcpu))  in vcpu_enter_guest()
10802 			process_nmi(vcpu);  in vcpu_enter_guest()
10803 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {  in vcpu_enter_guest()
10804 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);  in vcpu_enter_guest()
10805 			if (test_bit(vcpu->arch.pending_ioapic_eoi,  in vcpu_enter_guest()
10806 				     vcpu->arch.ioapic_handled_vectors)) {  in vcpu_enter_guest()
10807 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;  in vcpu_enter_guest()
10808 				vcpu->run->eoi.vector =  in vcpu_enter_guest()
10809 						vcpu->arch.pending_ioapic_eoi;  in vcpu_enter_guest()
10814 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))  in vcpu_enter_guest()
10815 			vcpu_scan_ioapic(vcpu);  in vcpu_enter_guest()
10816 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))  in vcpu_enter_guest()
10817 			vcpu_load_eoi_exitmap(vcpu);  in vcpu_enter_guest()
10818 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))  in vcpu_enter_guest()
10819 			kvm_vcpu_reload_apic_access_page(vcpu);  in vcpu_enter_guest()
10821 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {  in vcpu_enter_guest()
10822 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10823 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;  in vcpu_enter_guest()
10824 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10828 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {  in vcpu_enter_guest()
10829 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;  in vcpu_enter_guest()
10830 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;  in vcpu_enter_guest()
10831 			vcpu->run->system_event.ndata = 0;  in vcpu_enter_guest()
10835 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {  in vcpu_enter_guest()
10836 			struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);  in vcpu_enter_guest()
10838 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;  in vcpu_enter_guest()
10839 			vcpu->run->hyperv = hv_vcpu->exit;  in vcpu_enter_guest()
10849 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))  in vcpu_enter_guest()
10850 			kvm_hv_process_stimers(vcpu);  in vcpu_enter_guest()
10852 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))  in vcpu_enter_guest()
10853 			kvm_vcpu_update_apicv(vcpu);  in vcpu_enter_guest()
10854 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))  in vcpu_enter_guest()
10855 			kvm_check_async_pf_completion(vcpu);  in vcpu_enter_guest()
10856 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))  in vcpu_enter_guest()
10857 			kvm_x86_call(msr_filter_changed)(vcpu);  in vcpu_enter_guest()
10859 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))  in vcpu_enter_guest()
10860 			kvm_x86_call(update_cpu_dirty_logging)(vcpu);  in vcpu_enter_guest()
10862 		if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) {  in vcpu_enter_guest()
10863 			kvm_vcpu_reset(vcpu, true);  in vcpu_enter_guest()
10864 			if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE) {  in vcpu_enter_guest()
10871 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||  in vcpu_enter_guest()
10872 	    kvm_xen_has_interrupt(vcpu)) {  in vcpu_enter_guest()
10873 		++vcpu->stat.req_event;  in vcpu_enter_guest()
10874 		r = kvm_apic_accept_events(vcpu);  in vcpu_enter_guest()
10879 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {  in vcpu_enter_guest()
10884 		r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);  in vcpu_enter_guest()
10890 			kvm_x86_call(enable_irq_window)(vcpu);  in vcpu_enter_guest()
10892 		if (kvm_lapic_enabled(vcpu)) {  in vcpu_enter_guest()
10893 			update_cr8_intercept(vcpu);  in vcpu_enter_guest()
10894 			kvm_lapic_sync_to_vapic(vcpu);  in vcpu_enter_guest()
10898 	r = kvm_mmu_reload(vcpu);  in vcpu_enter_guest()
10905 	kvm_x86_call(prepare_switch_to_guest)(vcpu);  in vcpu_enter_guest()
10914 	/* Store vcpu->apicv_active before vcpu->mode.  */  in vcpu_enter_guest()
10915 	smp_store_release(&vcpu->mode, IN_GUEST_MODE);  in vcpu_enter_guest()
10917 	kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_enter_guest()
10928 	 * tables done while the VCPU is running.  Please see the comment  in vcpu_enter_guest()
10936 	 * target vCPU wasn't running).  Do this regardless of the vCPU's APICv  in vcpu_enter_guest()
10940 	if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10941 		kvm_x86_call(sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10943 	if (kvm_vcpu_exit_request(vcpu)) {  in vcpu_enter_guest()
10944 		vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
10948 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
10954 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
10960 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
10961 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);  in vcpu_enter_guest()
10963 	if (unlikely(vcpu->arch.switch_db_regs)) {  in vcpu_enter_guest()
10965 		set_debugreg(vcpu->arch.eff_db[0], 0);  in vcpu_enter_guest()
10966 		set_debugreg(vcpu->arch.eff_db[1], 1);  in vcpu_enter_guest()
10967 		set_debugreg(vcpu->arch.eff_db[2], 2);  in vcpu_enter_guest()
10968 		set_debugreg(vcpu->arch.eff_db[3], 3);  in vcpu_enter_guest()
10970 		if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))  in vcpu_enter_guest()
10971 			kvm_x86_call(set_dr6)(vcpu, vcpu->arch.dr6);  in vcpu_enter_guest()
10976 	vcpu->arch.host_debugctl = get_debugctlmsr();  in vcpu_enter_guest()
10982 		 * Assert that vCPU vs. VM APICv state is consistent.  An APICv  in vcpu_enter_guest()
10987 		WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&  in vcpu_enter_guest()
10988 			     (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));  in vcpu_enter_guest()
10990 		exit_fastpath = kvm_x86_call(vcpu_run)(vcpu,  in vcpu_enter_guest()
10995 		if (kvm_lapic_enabled(vcpu))  in vcpu_enter_guest()
10996 			kvm_x86_call(sync_pir_to_irr)(vcpu);  in vcpu_enter_guest()
10998 		if (unlikely(kvm_vcpu_exit_request(vcpu))) {  in vcpu_enter_guest()
11004 		++vcpu->stat.exits;  in vcpu_enter_guest()
11013 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {  in vcpu_enter_guest()
11014 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);  in vcpu_enter_guest()
11015 		kvm_x86_call(sync_dirty_debug_regs)(vcpu);  in vcpu_enter_guest()
11016 		kvm_update_dr0123(vcpu);  in vcpu_enter_guest()
11017 		kvm_update_dr7(vcpu);  in vcpu_enter_guest()
11030 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;  in vcpu_enter_guest()
11031 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());  in vcpu_enter_guest()
11033 	vcpu->mode = OUTSIDE_GUEST_MODE;  in vcpu_enter_guest()
11041 	if (vcpu->arch.xfd_no_write_intercept)  in vcpu_enter_guest()
11044 	kvm_x86_call(handle_exit_irqoff)(vcpu);  in vcpu_enter_guest()
11046 	if (vcpu->arch.guest_fpu.xfd_err)  in vcpu_enter_guest()
11056 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);  in vcpu_enter_guest()
11058 	++vcpu->stat.exits;  in vcpu_enter_guest()
11060 	kvm_after_interrupt(vcpu);  in vcpu_enter_guest()
11074 	kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_enter_guest()
11086 		unsigned long rip = kvm_rip_read(vcpu);  in vcpu_enter_guest()
11090 	if (unlikely(vcpu->arch.tsc_always_catchup))  in vcpu_enter_guest()
11091 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in vcpu_enter_guest()
11093 	if (vcpu->arch.apic_attention)  in vcpu_enter_guest()
11094 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
11099 	r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath);  in vcpu_enter_guest()
11104 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in vcpu_enter_guest()
11105 	kvm_x86_call(cancel_injection)(vcpu);  in vcpu_enter_guest()
11106 	if (unlikely(vcpu->arch.apic_attention))  in vcpu_enter_guest()
11107 		kvm_lapic_sync_from_vapic(vcpu);  in vcpu_enter_guest()
11112 static bool kvm_vcpu_running(struct kvm_vcpu *vcpu)  in kvm_vcpu_running()  argument
11114 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&  in kvm_vcpu_running()
11115 		!vcpu->arch.apf.halted);  in kvm_vcpu_running()
11118 static bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)  in kvm_vcpu_has_events()  argument
11120 	if (!list_empty_careful(&vcpu->async_pf.done))  in kvm_vcpu_has_events()
11123 	if (kvm_apic_has_pending_init_or_sipi(vcpu) &&  in kvm_vcpu_has_events()
11124 	    kvm_apic_init_sipi_allowed(vcpu))  in kvm_vcpu_has_events()
11127 	if (vcpu->arch.pv.pv_unhalted)  in kvm_vcpu_has_events()
11130 	if (kvm_is_exception_pending(vcpu))  in kvm_vcpu_has_events()
11133 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_vcpu_has_events()
11134 	    (vcpu->arch.nmi_pending &&  in kvm_vcpu_has_events()
11135 	     kvm_x86_call(nmi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
11139 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_vcpu_has_events()
11140 	    (vcpu->arch.smi_pending &&  in kvm_vcpu_has_events()
11141 	     kvm_x86_call(smi_allowed)(vcpu, false)))  in kvm_vcpu_has_events()
11145 	if (kvm_test_request(KVM_REQ_PMI, vcpu))  in kvm_vcpu_has_events()
11148 	if (kvm_test_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu))  in kvm_vcpu_has_events()
11151 	if (kvm_arch_interrupt_allowed(vcpu) && kvm_cpu_has_interrupt(vcpu))  in kvm_vcpu_has_events()
11154 	if (kvm_hv_has_stimer_pending(vcpu))  in kvm_vcpu_has_events()
11157 	if (is_guest_mode(vcpu) &&  in kvm_vcpu_has_events()
11159 	    kvm_x86_ops.nested_ops->has_events(vcpu, false))  in kvm_vcpu_has_events()
11162 	if (kvm_xen_has_pending_events(vcpu))  in kvm_vcpu_has_events()
11168 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_runnable()  argument
11170 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);  in kvm_arch_vcpu_runnable()
11174 static inline int vcpu_block(struct kvm_vcpu *vcpu)  in vcpu_block()  argument
11178 	if (!kvm_arch_vcpu_runnable(vcpu)) {  in vcpu_block()
11181 		 * the guest's timer may be a break event for the vCPU, and the  in vcpu_block()
11186 		hv_timer = kvm_lapic_hv_timer_in_use(vcpu);  in vcpu_block()
11188 			kvm_lapic_switch_to_sw_timer(vcpu);  in vcpu_block()
11190 		kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_block()
11191 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)  in vcpu_block()
11192 			kvm_vcpu_halt(vcpu);  in vcpu_block()
11194 			kvm_vcpu_block(vcpu);  in vcpu_block()
11195 		kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_block()
11198 			kvm_lapic_switch_to_hv_timer(vcpu);  in vcpu_block()
11201 		 * If the vCPU is not runnable, a signal or another host event  in vcpu_block()
11203 		 * vCPU's activity state.  in vcpu_block()
11205 		if (!kvm_arch_vcpu_runnable(vcpu))  in vcpu_block()
11215 	if (is_guest_mode(vcpu)) {  in vcpu_block()
11216 		int r = kvm_check_nested_events(vcpu);  in vcpu_block()
11223 	if (kvm_apic_accept_events(vcpu) < 0)  in vcpu_block()
11225 	switch(vcpu->arch.mp_state) {  in vcpu_block()
11228 		vcpu->arch.pv.pv_unhalted = false;  in vcpu_block()
11229 		vcpu->arch.mp_state =  in vcpu_block()
11233 		vcpu->arch.apf.halted = false;  in vcpu_block()
11245 static int vcpu_run(struct kvm_vcpu *vcpu)  in vcpu_run()  argument
11249 	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;  in vcpu_run()
11253 		 * If another guest vCPU requests a PV TLB flush in the middle  in vcpu_run()
11258 		vcpu->arch.at_instruction_boundary = false;  in vcpu_run()
11259 		if (kvm_vcpu_running(vcpu)) {  in vcpu_run()
11260 			r = vcpu_enter_guest(vcpu);  in vcpu_run()
11262 			r = vcpu_block(vcpu);  in vcpu_run()
11268 		kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);  in vcpu_run()
11269 		if (kvm_xen_has_pending_events(vcpu))  in vcpu_run()
11270 			kvm_xen_inject_pending_events(vcpu);  in vcpu_run()
11272 		if (kvm_cpu_has_pending_timer(vcpu))  in vcpu_run()
11273 			kvm_inject_pending_timer_irqs(vcpu);  in vcpu_run()
11275 		if (dm_request_for_irq_injection(vcpu) &&  in vcpu_run()
11276 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {  in vcpu_run()
11278 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;  in vcpu_run()
11279 			++vcpu->stat.request_irq_exits;  in vcpu_run()
11284 			kvm_vcpu_srcu_read_unlock(vcpu);  in vcpu_run()
11285 			r = xfer_to_guest_mode_handle_work(vcpu);  in vcpu_run()
11286 			kvm_vcpu_srcu_read_lock(vcpu);  in vcpu_run()
11295 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)  in __kvm_emulate_halt()  argument
11298 	 * The vCPU has halted, e.g. executed HLT.  Update the run state if the  in __kvm_emulate_halt()
11300 	 * state and halt the vCPU.  Exit to userspace if the local APIC is  in __kvm_emulate_halt()
11304 	++vcpu->stat.halt_exits;  in __kvm_emulate_halt()
11305 	if (lapic_in_kernel(vcpu)) {  in __kvm_emulate_halt()
11306 		if (kvm_vcpu_has_events(vcpu))  in __kvm_emulate_halt()
11307 			vcpu->arch.pv.pv_unhalted = false;  in __kvm_emulate_halt()
11309 			vcpu->arch.mp_state = state;  in __kvm_emulate_halt()
11312 		vcpu->run->exit_reason = reason;  in __kvm_emulate_halt()
11317 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)  in kvm_emulate_halt_noskip()  argument
11319 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);  in kvm_emulate_halt_noskip()
11323 int kvm_emulate_halt(struct kvm_vcpu *vcpu)  in kvm_emulate_halt()  argument
11325 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_halt()
11330 	return kvm_emulate_halt_noskip(vcpu) && ret;  in kvm_emulate_halt()
11334 fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu)  in handle_fastpath_hlt()  argument
11338 	kvm_vcpu_srcu_read_lock(vcpu);  in handle_fastpath_hlt()
11339 	ret = kvm_emulate_halt(vcpu);  in handle_fastpath_hlt()
11340 	kvm_vcpu_srcu_read_unlock(vcpu);  in handle_fastpath_hlt()
11345 	if (kvm_vcpu_running(vcpu))  in handle_fastpath_hlt()
11352 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)  in kvm_emulate_ap_reset_hold()  argument
11354 	int ret = kvm_skip_emulated_instruction(vcpu);  in kvm_emulate_ap_reset_hold()
11356 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,  in kvm_emulate_ap_reset_hold()
11361 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)  in kvm_arch_dy_has_pending_interrupt()  argument
11363 	return kvm_vcpu_apicv_active(vcpu) &&  in kvm_arch_dy_has_pending_interrupt()
11364 	       kvm_x86_call(dy_apicv_has_pending_interrupt)(vcpu);  in kvm_arch_dy_has_pending_interrupt()
11367 bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_preempted_in_kernel()  argument
11369 	return vcpu->arch.preempted_in_kernel;  in kvm_arch_vcpu_preempted_in_kernel()
11372 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_dy_runnable()  argument
11374 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))  in kvm_arch_dy_runnable()
11377 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||  in kvm_arch_dy_runnable()
11379 		kvm_test_request(KVM_REQ_SMI, vcpu) ||  in kvm_arch_dy_runnable()
11381 		 kvm_test_request(KVM_REQ_EVENT, vcpu))  in kvm_arch_dy_runnable()
11384 	return kvm_arch_dy_has_pending_interrupt(vcpu);  in kvm_arch_dy_runnable()
11387 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)  in complete_emulated_io()  argument
11389 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);  in complete_emulated_io()
11392 static int complete_emulated_pio(struct kvm_vcpu *vcpu)  in complete_emulated_pio()  argument
11394 	BUG_ON(!vcpu->arch.pio.count);  in complete_emulated_pio()
11396 	return complete_emulated_io(vcpu);  in complete_emulated_pio()
11417 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_emulated_mmio()  argument
11419 	struct kvm_run *run = vcpu->run;  in complete_emulated_mmio()
11423 	BUG_ON(!vcpu->mmio_needed);  in complete_emulated_mmio()
11426 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_emulated_mmio()
11428 	if (!vcpu->mmio_is_write)  in complete_emulated_mmio()
11434 		vcpu->mmio_cur_fragment++;  in complete_emulated_mmio()
11442 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_emulated_mmio()
11443 		vcpu->mmio_needed = 0;  in complete_emulated_mmio()
11446 		if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11448 		vcpu->mmio_read_completed = 1;  in complete_emulated_mmio()
11449 		return complete_emulated_io(vcpu);  in complete_emulated_mmio()
11454 	if (vcpu->mmio_is_write)  in complete_emulated_mmio()
11457 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_emulated_mmio()
11458 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;  in complete_emulated_mmio()
11463 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_load_guest_fpu()  argument
11466 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);  in kvm_load_guest_fpu()
11471 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)  in kvm_put_guest_fpu()  argument
11473 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);  in kvm_put_guest_fpu()
11474 	++vcpu->stat.fpu_reload;  in kvm_put_guest_fpu()
11478 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_ioctl_run()  argument
11480 	struct kvm_queued_exception *ex = &vcpu->arch.exception;  in kvm_arch_vcpu_ioctl_run()
11481 	struct kvm_run *kvm_run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
11485 	r = kvm_mmu_post_init_vm(vcpu->kvm);  in kvm_arch_vcpu_ioctl_run()
11489 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_run()
11490 	kvm_sigset_activate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11492 	kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11494 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11495 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {  in kvm_arch_vcpu_ioctl_run()
11496 		if (!vcpu->wants_to_run) {  in kvm_arch_vcpu_ioctl_run()
11504 		 * APIC timer to be active is if userspace stuffed vCPU state,  in kvm_arch_vcpu_ioctl_run()
11505 		 * i.e. put the vCPU into a nonsensical state.  Only an INIT  in kvm_arch_vcpu_ioctl_run()
11506 		 * will transition the vCPU out of UNINITIALIZED (without more  in kvm_arch_vcpu_ioctl_run()
11511 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11512 		kvm_vcpu_block(vcpu);  in kvm_arch_vcpu_ioctl_run()
11513 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11515 		if (kvm_apic_accept_events(vcpu) < 0) {  in kvm_arch_vcpu_ioctl_run()
11523 			++vcpu->stat.signal_exits;  in kvm_arch_vcpu_ioctl_run()
11528 	sync_valid_fields = kvm_sync_valid_fields(vcpu->kvm);  in kvm_arch_vcpu_ioctl_run()
11536 		r = sync_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11542 	if (!lapic_in_kernel(vcpu)) {  in kvm_arch_vcpu_ioctl_run()
11543 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {  in kvm_arch_vcpu_ioctl_run()
11553 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&  in kvm_arch_vcpu_ioctl_run()
11554 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11556 		kvm_queue_exception_vmexit(vcpu, ex->vector,  in kvm_arch_vcpu_ioctl_run()
11562 	vcpu->arch.exception_from_userspace = false;  in kvm_arch_vcpu_ioctl_run()
11564 	if (unlikely(vcpu->arch.complete_userspace_io)) {  in kvm_arch_vcpu_ioctl_run()
11565 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;  in kvm_arch_vcpu_ioctl_run()
11566 		vcpu->arch.complete_userspace_io = NULL;  in kvm_arch_vcpu_ioctl_run()
11567 		r = cui(vcpu);  in kvm_arch_vcpu_ioctl_run()
11571 		WARN_ON_ONCE(vcpu->arch.pio.count);  in kvm_arch_vcpu_ioctl_run()
11572 		WARN_ON_ONCE(vcpu->mmio_needed);  in kvm_arch_vcpu_ioctl_run()
11575 	if (!vcpu->wants_to_run) {  in kvm_arch_vcpu_ioctl_run()
11580 	r = kvm_x86_call(vcpu_pre_run)(vcpu);  in kvm_arch_vcpu_ioctl_run()
11584 	r = vcpu_run(vcpu);  in kvm_arch_vcpu_ioctl_run()
11587 	kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_run()
11588 	if (kvm_run->kvm_valid_regs && likely(!vcpu->arch.guest_state_protected))  in kvm_arch_vcpu_ioctl_run()
11589 		store_regs(vcpu);  in kvm_arch_vcpu_ioctl_run()
11590 	post_kvm_run_save(vcpu);  in kvm_arch_vcpu_ioctl_run()
11591 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
11593 	kvm_sigset_deactivate(vcpu);  in kvm_arch_vcpu_ioctl_run()
11594 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_run()
11598 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __get_regs()  argument
11600 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {  in __get_regs()
11604 		 * back from emulation context to vcpu. Userspace shouldn't do  in __get_regs()
11608 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);  in __get_regs()
11609 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __get_regs()
11611 	regs->rax = kvm_rax_read(vcpu);  in __get_regs()
11612 	regs->rbx = kvm_rbx_read(vcpu);  in __get_regs()
11613 	regs->rcx = kvm_rcx_read(vcpu);  in __get_regs()
11614 	regs->rdx = kvm_rdx_read(vcpu);  in __get_regs()
11615 	regs->rsi = kvm_rsi_read(vcpu);  in __get_regs()
11616 	regs->rdi = kvm_rdi_read(vcpu);  in __get_regs()
11617 	regs->rsp = kvm_rsp_read(vcpu);  in __get_regs()
11618 	regs->rbp = kvm_rbp_read(vcpu);  in __get_regs()
11620 	regs->r8 = kvm_r8_read(vcpu);  in __get_regs()
11621 	regs->r9 = kvm_r9_read(vcpu);  in __get_regs()
11622 	regs->r10 = kvm_r10_read(vcpu);  in __get_regs()
11623 	regs->r11 = kvm_r11_read(vcpu);  in __get_regs()
11624 	regs->r12 = kvm_r12_read(vcpu);  in __get_regs()
11625 	regs->r13 = kvm_r13_read(vcpu);  in __get_regs()
11626 	regs->r14 = kvm_r14_read(vcpu);  in __get_regs()
11627 	regs->r15 = kvm_r15_read(vcpu);  in __get_regs()
11630 	regs->rip = kvm_rip_read(vcpu);  in __get_regs()
11631 	regs->rflags = kvm_get_rflags(vcpu);  in __get_regs()
11634 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_get_regs()  argument
11636 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_arch_vcpu_ioctl_get_regs()
11637 	    vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_get_regs()
11640 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11641 	__get_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_get_regs()
11642 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_regs()
11646 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in __set_regs()  argument
11648 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;  in __set_regs()
11649 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;  in __set_regs()
11651 	kvm_rax_write(vcpu, regs->rax);  in __set_regs()
11652 	kvm_rbx_write(vcpu, regs->rbx);  in __set_regs()
11653 	kvm_rcx_write(vcpu, regs->rcx);  in __set_regs()
11654 	kvm_rdx_write(vcpu, regs->rdx);  in __set_regs()
11655 	kvm_rsi_write(vcpu, regs->rsi);  in __set_regs()
11656 	kvm_rdi_write(vcpu, regs->rdi);  in __set_regs()
11657 	kvm_rsp_write(vcpu, regs->rsp);  in __set_regs()
11658 	kvm_rbp_write(vcpu, regs->rbp);  in __set_regs()
11660 	kvm_r8_write(vcpu, regs->r8);  in __set_regs()
11661 	kvm_r9_write(vcpu, regs->r9);  in __set_regs()
11662 	kvm_r10_write(vcpu, regs->r10);  in __set_regs()
11663 	kvm_r11_write(vcpu, regs->r11);  in __set_regs()
11664 	kvm_r12_write(vcpu, regs->r12);  in __set_regs()
11665 	kvm_r13_write(vcpu, regs->r13);  in __set_regs()
11666 	kvm_r14_write(vcpu, regs->r14);  in __set_regs()
11667 	kvm_r15_write(vcpu, regs->r15);  in __set_regs()
11670 	kvm_rip_write(vcpu, regs->rip);  in __set_regs()
11671 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);  in __set_regs()
11673 	vcpu->arch.exception.pending = false;  in __set_regs()
11674 	vcpu->arch.exception_vmexit.pending = false;  in __set_regs()
11676 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_regs()
11679 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_set_regs()  argument
11681 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_arch_vcpu_ioctl_set_regs()
11682 	    vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_regs()
11685 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11686 	__set_regs(vcpu, regs);  in kvm_arch_vcpu_ioctl_set_regs()
11687 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_regs()
11691 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs_common()  argument
11695 	if (vcpu->arch.guest_state_protected)  in __get_sregs_common()
11698 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __get_sregs_common()
11699 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __get_sregs_common()
11700 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __get_sregs_common()
11701 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __get_sregs_common()
11702 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __get_sregs_common()
11703 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __get_sregs_common()
11705 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __get_sregs_common()
11706 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __get_sregs_common()
11708 	kvm_x86_call(get_idt)(vcpu, &dt);  in __get_sregs_common()
11711 	kvm_x86_call(get_gdt)(vcpu, &dt);  in __get_sregs_common()
11715 	sregs->cr2 = vcpu->arch.cr2;  in __get_sregs_common()
11716 	sregs->cr3 = kvm_read_cr3(vcpu);  in __get_sregs_common()
11719 	sregs->cr0 = kvm_read_cr0(vcpu);  in __get_sregs_common()
11720 	sregs->cr4 = kvm_read_cr4(vcpu);  in __get_sregs_common()
11721 	sregs->cr8 = kvm_get_cr8(vcpu);  in __get_sregs_common()
11722 	sregs->efer = vcpu->arch.efer;  in __get_sregs_common()
11723 	sregs->apic_base = vcpu->arch.apic_base;  in __get_sregs_common()
11726 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __get_sregs()  argument
11728 	__get_sregs_common(vcpu, sregs);  in __get_sregs()
11730 	if (vcpu->arch.guest_state_protected)  in __get_sregs()
11733 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)  in __get_sregs()
11734 		set_bit(vcpu->arch.interrupt.nr,  in __get_sregs()
11738 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __get_sregs2()  argument
11742 	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);  in __get_sregs2()
11744 	if (vcpu->arch.guest_state_protected)  in __get_sregs2()
11747 	if (is_pae_paging(vcpu)) {  in __get_sregs2()
11749 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);  in __get_sregs2()
11754 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_sregs()  argument
11757 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_arch_vcpu_ioctl_get_sregs()
11758 	    vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_get_sregs()
11761 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11762 	__get_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_get_sregs()
11763 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_sregs()
11767 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_mpstate()  argument
11772 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11774 		kvm_load_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11776 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11778 	r = kvm_apic_accept_events(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11783 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||  in kvm_arch_vcpu_ioctl_get_mpstate()
11784 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&  in kvm_arch_vcpu_ioctl_get_mpstate()
11785 	    vcpu->arch.pv.pv_unhalted)  in kvm_arch_vcpu_ioctl_get_mpstate()
11788 		mp_state->mp_state = vcpu->arch.mp_state;  in kvm_arch_vcpu_ioctl_get_mpstate()
11791 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11794 		kvm_put_guest_fpu(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11795 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_mpstate()
11799 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_mpstate()  argument
11804 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11812 		if (!lapic_in_kernel(vcpu))  in kvm_arch_vcpu_ioctl_set_mpstate()
11829 	if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&  in kvm_arch_vcpu_ioctl_set_mpstate()
11835 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;  in kvm_arch_vcpu_ioctl_set_mpstate()
11836 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);  in kvm_arch_vcpu_ioctl_set_mpstate()
11838 		vcpu->arch.mp_state = mp_state->mp_state;  in kvm_arch_vcpu_ioctl_set_mpstate()
11839 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11843 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
11847 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,  in kvm_task_switch()  argument
11850 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;  in kvm_task_switch()
11853 	init_emulate_ctxt(vcpu);  in kvm_task_switch()
11862 	if (ret || vcpu->mmio_needed) {  in kvm_task_switch()
11863 		vcpu->mmio_needed = false;  in kvm_task_switch()
11864 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;  in kvm_task_switch()
11865 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;  in kvm_task_switch()
11866 		vcpu->run->internal.ndata = 0;  in kvm_task_switch()
11870 	kvm_rip_write(vcpu, ctxt->eip);  in kvm_task_switch()
11871 	kvm_set_rflags(vcpu, ctxt->eflags);  in kvm_task_switch()
11876 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in kvm_is_valid_sregs()  argument
11886 		if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3))  in kvm_is_valid_sregs()
11897 	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&  in kvm_is_valid_sregs()
11898 	       kvm_is_valid_cr0(vcpu, sregs->cr0);  in kvm_is_valid_sregs()
11901 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,  in __set_sregs_common()  argument
11907 	if (!kvm_is_valid_sregs(vcpu, sregs))  in __set_sregs_common()
11910 	if (kvm_apic_set_base(vcpu, sregs->apic_base, true))  in __set_sregs_common()
11913 	if (vcpu->arch.guest_state_protected)  in __set_sregs_common()
11918 	kvm_x86_call(set_idt)(vcpu, &dt);  in __set_sregs_common()
11921 	kvm_x86_call(set_gdt)(vcpu, &dt);  in __set_sregs_common()
11923 	vcpu->arch.cr2 = sregs->cr2;  in __set_sregs_common()
11924 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;  in __set_sregs_common()
11925 	vcpu->arch.cr3 = sregs->cr3;  in __set_sregs_common()
11926 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in __set_sregs_common()
11927 	kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3);  in __set_sregs_common()
11929 	kvm_set_cr8(vcpu, sregs->cr8);  in __set_sregs_common()
11931 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;  in __set_sregs_common()
11932 	kvm_x86_call(set_efer)(vcpu, sregs->efer);  in __set_sregs_common()
11934 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;  in __set_sregs_common()
11935 	kvm_x86_call(set_cr0)(vcpu, sregs->cr0);  in __set_sregs_common()
11937 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;  in __set_sregs_common()
11938 	kvm_x86_call(set_cr4)(vcpu, sregs->cr4);  in __set_sregs_common()
11941 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in __set_sregs_common()
11942 		if (is_pae_paging(vcpu)) {  in __set_sregs_common()
11943 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));  in __set_sregs_common()
11946 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in __set_sregs_common()
11949 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);  in __set_sregs_common()
11950 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);  in __set_sregs_common()
11951 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);  in __set_sregs_common()
11952 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);  in __set_sregs_common()
11953 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);  in __set_sregs_common()
11954 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);  in __set_sregs_common()
11956 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);  in __set_sregs_common()
11957 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);  in __set_sregs_common()
11959 	update_cr8_intercept(vcpu);  in __set_sregs_common()
11961 	/* Older userspace won't unhalt the vcpu on reset. */  in __set_sregs_common()
11962 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&  in __set_sregs_common()
11964 	    !is_protmode(vcpu))  in __set_sregs_common()
11965 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in __set_sregs_common()
11970 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)  in __set_sregs()  argument
11974 	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);  in __set_sregs()
11980 		kvm_mmu_reset_context(vcpu);  in __set_sregs()
11981 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in __set_sregs()
11989 		kvm_queue_interrupt(vcpu, pending_vec, false);  in __set_sregs()
11991 		kvm_make_request(KVM_REQ_EVENT, vcpu);  in __set_sregs()
11996 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)  in __set_sregs2()  argument
12007 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))  in __set_sregs2()
12010 	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,  in __set_sregs2()
12017 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);  in __set_sregs2()
12019 		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);  in __set_sregs2()
12021 		vcpu->arch.pdptrs_from_userspace = true;  in __set_sregs2()
12024 		kvm_mmu_reset_context(vcpu);  in __set_sregs2()
12025 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in __set_sregs2()
12030 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_sregs()  argument
12035 	if (vcpu->kvm->arch.has_protected_state &&  in kvm_arch_vcpu_ioctl_set_sregs()
12036 	    vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_sregs()
12039 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
12040 	ret = __set_sregs(vcpu, sregs);  in kvm_arch_vcpu_ioctl_set_sregs()
12041 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_sregs()
12048 	struct kvm_vcpu *vcpu;  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()  local
12056 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
12057 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {  in kvm_arch_vcpu_guestdbg_update_apicv_inhibit()
12066 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_guest_debug()  argument
12072 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_ioctl_set_guest_debug()
12075 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12079 		if (kvm_is_exception_pending(vcpu))  in kvm_arch_vcpu_ioctl_set_guest_debug()
12082 			kvm_queue_exception(vcpu, DB_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12084 			kvm_queue_exception(vcpu, BP_VECTOR);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12091 	rflags = kvm_get_rflags(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12093 	vcpu->guest_debug = dbg->control;  in kvm_arch_vcpu_ioctl_set_guest_debug()
12094 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))  in kvm_arch_vcpu_ioctl_set_guest_debug()
12095 		vcpu->guest_debug = 0;  in kvm_arch_vcpu_ioctl_set_guest_debug()
12097 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {  in kvm_arch_vcpu_ioctl_set_guest_debug()
12099 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
12100 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];  in kvm_arch_vcpu_ioctl_set_guest_debug()
12103 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];  in kvm_arch_vcpu_ioctl_set_guest_debug()
12105 	kvm_update_dr7(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12107 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_arch_vcpu_ioctl_set_guest_debug()
12108 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12114 	kvm_set_rflags(vcpu, rflags);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12116 	kvm_x86_call(update_exception_bitmap)(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12118 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12123 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_guest_debug()
12130 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_translate()  argument
12137 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_translate()
12139 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_ioctl_translate()
12140 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);  in kvm_arch_vcpu_ioctl_translate()
12141 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_ioctl_translate()
12147 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_translate()
12151 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_get_fpu()  argument
12155 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_get_fpu()
12156 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;  in kvm_arch_vcpu_ioctl_get_fpu()
12158 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
12160 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_get_fpu()
12170 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_get_fpu()
12174 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_set_fpu()  argument
12178 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))  in kvm_arch_vcpu_ioctl_set_fpu()
12179 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;  in kvm_arch_vcpu_ioctl_set_fpu()
12181 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
12183 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;  in kvm_arch_vcpu_ioctl_set_fpu()
12194 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_set_fpu()
12198 static void store_regs(struct kvm_vcpu *vcpu)  in store_regs()  argument
12202 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)  in store_regs()
12203 		__get_regs(vcpu, &vcpu->run->s.regs.regs);  in store_regs()
12205 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)  in store_regs()
12206 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);  in store_regs()
12208 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)  in store_regs()
12210 				vcpu, &vcpu->run->s.regs.events);  in store_regs()
12213 static int sync_regs(struct kvm_vcpu *vcpu)  in sync_regs()  argument
12215 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {  in sync_regs()
12216 		__set_regs(vcpu, &vcpu->run->s.regs.regs);  in sync_regs()
12217 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;  in sync_regs()
12220 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {  in sync_regs()
12221 		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;  in sync_regs()
12223 		if (__set_sregs(vcpu, &sregs))  in sync_regs()
12226 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;  in sync_regs()
12229 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {  in sync_regs()
12230 		struct kvm_vcpu_events events = vcpu->run->s.regs.events;  in sync_regs()
12232 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events))  in sync_regs()
12235 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;  in sync_regs()
12256 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_create()  argument
12261 	vcpu->arch.last_vmentry_cpu = -1;  in kvm_arch_vcpu_create()
12262 	vcpu->arch.regs_avail = ~0;  in kvm_arch_vcpu_create()
12263 	vcpu->arch.regs_dirty = ~0;  in kvm_arch_vcpu_create()
12265 	kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm);  in kvm_arch_vcpu_create()
12267 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))  in kvm_arch_vcpu_create()
12268 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_vcpu_create()
12270 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;  in kvm_arch_vcpu_create()
12272 	r = kvm_mmu_create(vcpu);  in kvm_arch_vcpu_create()
12276 	r = kvm_create_lapic(vcpu);  in kvm_arch_vcpu_create()
12285 	vcpu->arch.pio_data = page_address(page);  in kvm_arch_vcpu_create()
12287 	vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),  in kvm_arch_vcpu_create()
12289 	vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),  in kvm_arch_vcpu_create()
12291 	if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)  in kvm_arch_vcpu_create()
12293 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;  in kvm_arch_vcpu_create()
12295 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,  in kvm_arch_vcpu_create()
12299 	if (!alloc_emulate_ctxt(vcpu))  in kvm_arch_vcpu_create()
12302 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {  in kvm_arch_vcpu_create()
12303 		pr_err("failed to allocate vcpu's fpu\n");  in kvm_arch_vcpu_create()
12307 	kvm_async_pf_hash_reset(vcpu);  in kvm_arch_vcpu_create()
12309 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) {  in kvm_arch_vcpu_create()
12310 		vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();  in kvm_arch_vcpu_create()
12311 		vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;  in kvm_arch_vcpu_create()
12312 		vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;  in kvm_arch_vcpu_create()
12314 	kvm_pmu_init(vcpu);  in kvm_arch_vcpu_create()
12316 	vcpu->arch.pending_external_vector = -1;  in kvm_arch_vcpu_create()
12317 	vcpu->arch.preempted_in_kernel = false;  in kvm_arch_vcpu_create()
12320 	vcpu->arch.hv_root_tdp = INVALID_PAGE;  in kvm_arch_vcpu_create()
12323 	r = kvm_x86_call(vcpu_create)(vcpu);  in kvm_arch_vcpu_create()
12327 	kvm_xen_init_vcpu(vcpu);  in kvm_arch_vcpu_create()
12328 	vcpu_load(vcpu);  in kvm_arch_vcpu_create()
12329 	kvm_vcpu_after_set_cpuid(vcpu);  in kvm_arch_vcpu_create()
12330 	kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);  in kvm_arch_vcpu_create()
12331 	kvm_vcpu_reset(vcpu, false);  in kvm_arch_vcpu_create()
12332 	kvm_init_mmu(vcpu);  in kvm_arch_vcpu_create()
12333 	vcpu_put(vcpu);  in kvm_arch_vcpu_create()
12337 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_create()
12339 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_create()
12341 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_create()
12343 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_create()
12344 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_create()
12345 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_create()
12347 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_create()
12349 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_create()
12353 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_postcreate()  argument
12355 	struct kvm *kvm = vcpu->kvm;  in kvm_arch_vcpu_postcreate()
12357 	if (mutex_lock_killable(&vcpu->mutex))  in kvm_arch_vcpu_postcreate()
12359 	vcpu_load(vcpu);  in kvm_arch_vcpu_postcreate()
12360 	kvm_synchronize_tsc(vcpu, NULL);  in kvm_arch_vcpu_postcreate()
12361 	vcpu_put(vcpu);  in kvm_arch_vcpu_postcreate()
12364 	vcpu->arch.msr_kvm_poll_control = 1;  in kvm_arch_vcpu_postcreate()
12366 	mutex_unlock(&vcpu->mutex);  in kvm_arch_vcpu_postcreate()
12368 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)  in kvm_arch_vcpu_postcreate()
12373 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_destroy()  argument
12377 	kvmclock_reset(vcpu);  in kvm_arch_vcpu_destroy()
12379 	kvm_x86_call(vcpu_free)(vcpu);  in kvm_arch_vcpu_destroy()
12381 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);  in kvm_arch_vcpu_destroy()
12382 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);  in kvm_arch_vcpu_destroy()
12383 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);  in kvm_arch_vcpu_destroy()
12385 	kvm_xen_destroy_vcpu(vcpu);  in kvm_arch_vcpu_destroy()
12386 	kvm_hv_vcpu_uninit(vcpu);  in kvm_arch_vcpu_destroy()
12387 	kvm_pmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12388 	kfree(vcpu->arch.mce_banks);  in kvm_arch_vcpu_destroy()
12389 	kfree(vcpu->arch.mci_ctl2_banks);  in kvm_arch_vcpu_destroy()
12390 	kvm_free_lapic(vcpu);  in kvm_arch_vcpu_destroy()
12391 	idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvm_arch_vcpu_destroy()
12392 	kvm_mmu_destroy(vcpu);  in kvm_arch_vcpu_destroy()
12393 	srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvm_arch_vcpu_destroy()
12394 	free_page((unsigned long)vcpu->arch.pio_data);  in kvm_arch_vcpu_destroy()
12395 	kvfree(vcpu->arch.cpuid_entries);  in kvm_arch_vcpu_destroy()
12398 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)  in kvm_vcpu_reset()  argument
12401 	unsigned long old_cr0 = kvm_read_cr0(vcpu);  in kvm_vcpu_reset()
12408 	 * into hardware, to be zeroed at vCPU creation.  Use CRs as a sentinel  in kvm_vcpu_reset()
12412 		     (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));  in kvm_vcpu_reset()
12416 	 * possible to INIT the vCPU while L2 is active.  Force the vCPU back  in kvm_vcpu_reset()
12420 	if (is_guest_mode(vcpu))  in kvm_vcpu_reset()
12421 		kvm_leave_nested(vcpu);  in kvm_vcpu_reset()
12423 	kvm_lapic_reset(vcpu, init_event);  in kvm_vcpu_reset()
12425 	WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));  in kvm_vcpu_reset()
12426 	vcpu->arch.hflags = 0;  in kvm_vcpu_reset()
12428 	vcpu->arch.smi_pending = 0;  in kvm_vcpu_reset()
12429 	vcpu->arch.smi_count = 0;  in kvm_vcpu_reset()
12430 	atomic_set(&vcpu->arch.nmi_queued, 0);  in kvm_vcpu_reset()
12431 	vcpu->arch.nmi_pending = 0;  in kvm_vcpu_reset()
12432 	vcpu->arch.nmi_injected = false;  in kvm_vcpu_reset()
12433 	kvm_clear_interrupt_queue(vcpu);  in kvm_vcpu_reset()
12434 	kvm_clear_exception_queue(vcpu);  in kvm_vcpu_reset()
12436 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));  in kvm_vcpu_reset()
12437 	kvm_update_dr0123(vcpu);  in kvm_vcpu_reset()
12438 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;  in kvm_vcpu_reset()
12439 	vcpu->arch.dr7 = DR7_FIXED_1;  in kvm_vcpu_reset()
12440 	kvm_update_dr7(vcpu);  in kvm_vcpu_reset()
12442 	vcpu->arch.cr2 = 0;  in kvm_vcpu_reset()
12444 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_vcpu_reset()
12445 	vcpu->arch.apf.msr_en_val = 0;  in kvm_vcpu_reset()
12446 	vcpu->arch.apf.msr_int_val = 0;  in kvm_vcpu_reset()
12447 	vcpu->arch.st.msr_val = 0;  in kvm_vcpu_reset()
12449 	kvmclock_reset(vcpu);  in kvm_vcpu_reset()
12451 	kvm_clear_async_pf_completion_queue(vcpu);  in kvm_vcpu_reset()
12452 	kvm_async_pf_hash_reset(vcpu);  in kvm_vcpu_reset()
12453 	vcpu->arch.apf.halted = false;  in kvm_vcpu_reset()
12455 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {  in kvm_vcpu_reset()
12456 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;  in kvm_vcpu_reset()
12463 			kvm_put_guest_fpu(vcpu);  in kvm_vcpu_reset()
12469 			kvm_load_guest_fpu(vcpu);  in kvm_vcpu_reset()
12473 		vcpu->arch.smbase = 0x30000;  in kvm_vcpu_reset()
12475 		vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;  in kvm_vcpu_reset()
12477 		vcpu->arch.msr_misc_features_enables = 0;  in kvm_vcpu_reset()
12478 		vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |  in kvm_vcpu_reset()
12481 		__kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);  in kvm_vcpu_reset()
12482 		__kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);  in kvm_vcpu_reset()
12486 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));  in kvm_vcpu_reset()
12487 	kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);  in kvm_vcpu_reset()
12492 	 * RESET since KVM emulates RESET before exposing the vCPU to userspace,  in kvm_vcpu_reset()
12496 	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);  in kvm_vcpu_reset()
12497 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);  in kvm_vcpu_reset()
12499 	kvm_x86_call(vcpu_reset)(vcpu, init_event);  in kvm_vcpu_reset()
12501 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);  in kvm_vcpu_reset()
12502 	kvm_rip_write(vcpu, 0xfff0);  in kvm_vcpu_reset()
12504 	vcpu->arch.cr3 = 0;  in kvm_vcpu_reset()
12505 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);  in kvm_vcpu_reset()
12518 	kvm_x86_call(set_cr0)(vcpu, new_cr0);  in kvm_vcpu_reset()
12519 	kvm_x86_call(set_cr4)(vcpu, 0);  in kvm_vcpu_reset()
12520 	kvm_x86_call(set_efer)(vcpu, 0);  in kvm_vcpu_reset()
12521 	kvm_x86_call(update_exception_bitmap)(vcpu);  in kvm_vcpu_reset()
12532 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12533 		kvm_mmu_reset_context(vcpu);  in kvm_vcpu_reset()
12546 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_vcpu_reset()
12550 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)  in kvm_vcpu_deliver_sipi_vector()  argument
12554 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12557 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);  in kvm_vcpu_deliver_sipi_vector()
12558 	kvm_rip_write(vcpu, 0);  in kvm_vcpu_deliver_sipi_vector()
12575 	struct kvm_vcpu *vcpu;  in kvm_arch_enable_virtualization_cpu()  local
12595 		kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_enable_virtualization_cpu()
12596 			if (!stable && vcpu->cpu == smp_processor_id())  in kvm_arch_enable_virtualization_cpu()
12597 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);  in kvm_arch_enable_virtualization_cpu()
12598 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {  in kvm_arch_enable_virtualization_cpu()
12600 				if (vcpu->arch.last_host_tsc > max_tsc)  in kvm_arch_enable_virtualization_cpu()
12601 					max_tsc = vcpu->arch.last_host_tsc;  in kvm_arch_enable_virtualization_cpu()
12618 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,  in kvm_arch_enable_virtualization_cpu()
12620 	 * adjustments, in case multiple suspend cycles happen before some VCPU  in kvm_arch_enable_virtualization_cpu()
12648 			kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_arch_enable_virtualization_cpu()
12649 				vcpu->arch.tsc_offset_adjustment += delta_cyc;  in kvm_arch_enable_virtualization_cpu()
12650 				vcpu->arch.last_host_tsc = local_tsc;  in kvm_arch_enable_virtualization_cpu()
12651 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);  in kvm_arch_enable_virtualization_cpu()
12674 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_reset_bsp()  argument
12676 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;  in kvm_vcpu_is_reset_bsp()
12679 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)  in kvm_vcpu_is_bsp()  argument
12681 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;  in kvm_vcpu_is_bsp()
12770 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)  in kvm_unload_vcpu_mmu()  argument
12772 	vcpu_load(vcpu);  in kvm_unload_vcpu_mmu()
12773 	kvm_mmu_unload(vcpu);  in kvm_unload_vcpu_mmu()
12774 	vcpu_put(vcpu);  in kvm_unload_vcpu_mmu()
12780 	struct kvm_vcpu *vcpu;  in kvm_unload_vcpu_mmus()  local
12782 	kvm_for_each_vcpu(i, vcpu, kvm) {  in kvm_unload_vcpu_mmus()
12783 		kvm_clear_async_pf_completion_queue(vcpu);  in kvm_unload_vcpu_mmus()
12784 		kvm_unload_vcpu_mmu(vcpu);  in kvm_unload_vcpu_mmus()
13023 	struct kvm_vcpu *vcpu;  in kvm_arch_memslots_updated()  local
13033 	kvm_for_each_vcpu(i, vcpu, kvm)  in kvm_arch_memslots_updated()
13034 		kvm_vcpu_kick(vcpu);  in kvm_arch_memslots_updated()
13221 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_in_kernel()  argument
13223 	WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu));  in kvm_arch_vcpu_in_kernel()
13225 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_in_kernel()
13228 	return kvm_x86_call(get_cpl)(vcpu) == 0;  in kvm_arch_vcpu_in_kernel()
13231 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_get_ip()  argument
13233 	WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu));  in kvm_arch_vcpu_get_ip()
13235 	if (vcpu->arch.guest_state_protected)  in kvm_arch_vcpu_get_ip()
13238 	return kvm_rip_read(vcpu);  in kvm_arch_vcpu_get_ip()
13241 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_should_kick()  argument
13243 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;  in kvm_arch_vcpu_should_kick()
13246 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)  in kvm_arch_interrupt_allowed()  argument
13248 	return kvm_x86_call(interrupt_allowed)(vcpu, false);  in kvm_arch_interrupt_allowed()
13251 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)  in kvm_get_linear_rip()  argument
13254 	if (vcpu->arch.guest_state_protected)  in kvm_get_linear_rip()
13257 	if (is_64_bit_mode(vcpu))  in kvm_get_linear_rip()
13258 		return kvm_rip_read(vcpu);  in kvm_get_linear_rip()
13259 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +  in kvm_get_linear_rip()
13260 		     kvm_rip_read(vcpu));  in kvm_get_linear_rip()
13264 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)  in kvm_is_linear_rip()  argument
13266 	return kvm_get_linear_rip(vcpu) == linear_rip;  in kvm_is_linear_rip()
13270 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)  in kvm_get_rflags()  argument
13274 	rflags = kvm_x86_call(get_rflags)(vcpu);  in kvm_get_rflags()
13275 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)  in kvm_get_rflags()
13281 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in __kvm_set_rflags()  argument
13283 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&  in __kvm_set_rflags()
13284 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))  in __kvm_set_rflags()
13286 	kvm_x86_call(set_rflags)(vcpu, rflags);  in __kvm_set_rflags()
13289 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)  in kvm_set_rflags()  argument
13291 	__kvm_set_rflags(vcpu, rflags);  in kvm_set_rflags()
13292 	kvm_make_request(KVM_REQ_EVENT, vcpu);  in kvm_set_rflags()
13308 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_add_async_pf_gfn()  argument
13312 	while (vcpu->arch.apf.gfns[key] != ~0)  in kvm_add_async_pf_gfn()
13315 	vcpu->arch.apf.gfns[key] = gfn;  in kvm_add_async_pf_gfn()
13318 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_async_pf_gfn_slot()  argument
13324 		     (vcpu->arch.apf.gfns[key] != gfn &&  in kvm_async_pf_gfn_slot()
13325 		      vcpu->arch.apf.gfns[key] != ~0); i++)  in kvm_async_pf_gfn_slot()
13331 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_find_async_pf_gfn()  argument
13333 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;  in kvm_find_async_pf_gfn()
13336 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_del_async_pf_gfn()  argument
13340 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);  in kvm_del_async_pf_gfn()
13342 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))  in kvm_del_async_pf_gfn()
13346 		vcpu->arch.apf.gfns[i] = ~0;  in kvm_del_async_pf_gfn()
13349 			if (vcpu->arch.apf.gfns[j] == ~0)  in kvm_del_async_pf_gfn()
13351 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);  in kvm_del_async_pf_gfn()
13358 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];  in kvm_del_async_pf_gfn()
13363 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)  in apf_put_user_notpresent()  argument
13367 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,  in apf_put_user_notpresent()
13371 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)  in apf_put_user_ready()  argument
13375 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_put_user_ready()
13379 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)  in apf_pageready_slot_free()  argument
13384 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,  in apf_pageready_slot_free()
13391 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_deliver_async_pf()  argument
13394 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_can_deliver_async_pf()
13397 	if (vcpu->arch.apf.send_user_only &&  in kvm_can_deliver_async_pf()
13398 	    kvm_x86_call(get_cpl)(vcpu) == 0)  in kvm_can_deliver_async_pf()
13401 	if (is_guest_mode(vcpu)) {  in kvm_can_deliver_async_pf()
13406 		return vcpu->arch.apf.delivery_as_pf_vmexit;  in kvm_can_deliver_async_pf()
13413 		return is_paging(vcpu);  in kvm_can_deliver_async_pf()
13417 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)  in kvm_can_do_async_pf()  argument
13419 	if (unlikely(!lapic_in_kernel(vcpu) ||  in kvm_can_do_async_pf()
13420 		     kvm_event_needs_reinjection(vcpu) ||  in kvm_can_do_async_pf()
13421 		     kvm_is_exception_pending(vcpu)))  in kvm_can_do_async_pf()
13424 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))  in kvm_can_do_async_pf()
13431 	return kvm_arch_interrupt_allowed(vcpu);  in kvm_can_do_async_pf()
13434 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_not_present()  argument
13440 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_not_present()
13442 	if (kvm_can_deliver_async_pf(vcpu) &&  in kvm_arch_async_page_not_present()
13443 	    !apf_put_user_notpresent(vcpu)) {  in kvm_arch_async_page_not_present()
13450 		kvm_inject_page_fault(vcpu, &fault);  in kvm_arch_async_page_not_present()
13461 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);  in kvm_arch_async_page_not_present()
13466 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,  in kvm_arch_async_page_present()  argument
13471 		.vector = vcpu->arch.apf.vec  in kvm_arch_async_page_present()
13477 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);  in kvm_arch_async_page_present()
13481 	    kvm_pv_async_pf_enabled(vcpu) &&  in kvm_arch_async_page_present()
13482 	    !apf_put_user_ready(vcpu, work->arch.token)) {  in kvm_arch_async_page_present()
13483 		vcpu->arch.apf.pageready_pending = true;  in kvm_arch_async_page_present()
13484 		kvm_apic_set_irq(vcpu, &irq, NULL);  in kvm_arch_async_page_present()
13487 	vcpu->arch.apf.halted = false;  in kvm_arch_async_page_present()
13488 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;  in kvm_arch_async_page_present()
13491 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)  in kvm_arch_async_page_present_queued()  argument
13493 	kvm_make_request(KVM_REQ_APF_READY, vcpu);  in kvm_arch_async_page_present_queued()
13494 	if (!vcpu->arch.apf.pageready_pending)  in kvm_arch_async_page_present_queued()
13495 		kvm_vcpu_kick(vcpu);  in kvm_arch_async_page_present_queued()
13498 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)  in kvm_arch_can_dequeue_async_page_present()  argument
13500 	if (!kvm_pv_async_pf_enabled(vcpu))  in kvm_arch_can_dequeue_async_page_present()
13503 		return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);  in kvm_arch_can_dequeue_async_page_present()
13625 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)  in kvm_arch_no_poll()  argument
13627 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;  in kvm_arch_no_poll()
13671 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)  in kvm_fixup_and_inject_pf_error()  argument
13673 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;  in kvm_fixup_and_inject_pf_error()
13679 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {  in kvm_fixup_and_inject_pf_error()
13681 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page  in kvm_fixup_and_inject_pf_error()
13692 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);  in kvm_fixup_and_inject_pf_error()
13701 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,  in kvm_handle_memory_failure()  argument
13705 		if (KVM_BUG_ON(!e, vcpu->kvm))  in kvm_handle_memory_failure()
13708 		kvm_inject_emulated_page_fault(vcpu, e);  in kvm_handle_memory_failure()
13719 	kvm_prepare_emulation_failure_exit(vcpu);  in kvm_handle_memory_failure()
13725 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)  in kvm_handle_invpcid()  argument
13735 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);  in kvm_handle_invpcid()
13737 		return kvm_handle_memory_failure(vcpu, r, &e);  in kvm_handle_invpcid()
13740 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13744 	pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);  in kvm_handle_invpcid()
13753 		    is_noncanonical_invlpg_address(operand.gla, vcpu)) {  in kvm_handle_invpcid()
13754 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13757 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);  in kvm_handle_invpcid()
13758 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13762 			kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13766 		kvm_invalidate_pcid(vcpu, operand.pcid);  in kvm_handle_invpcid()
13767 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13779 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);  in kvm_handle_invpcid()
13780 		return kvm_skip_emulated_instruction(vcpu);  in kvm_handle_invpcid()
13783 		kvm_inject_gp(vcpu, 0);  in kvm_handle_invpcid()
13789 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_mmio()  argument
13791 	struct kvm_run *run = vcpu->run;  in complete_sev_es_emulated_mmio()
13795 	BUG_ON(!vcpu->mmio_needed);  in complete_sev_es_emulated_mmio()
13798 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];  in complete_sev_es_emulated_mmio()
13800 	if (!vcpu->mmio_is_write)  in complete_sev_es_emulated_mmio()
13806 		vcpu->mmio_cur_fragment++;  in complete_sev_es_emulated_mmio()
13814 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {  in complete_sev_es_emulated_mmio()
13815 		vcpu->mmio_needed = 0;  in complete_sev_es_emulated_mmio()
13825 	run->mmio.is_write = vcpu->mmio_is_write;  in complete_sev_es_emulated_mmio()
13830 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in complete_sev_es_emulated_mmio()
13835 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_write()  argument
13844 	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_write()
13853 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_write()
13854 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_write()
13859 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_write()
13860 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_write()
13862 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_write()
13863 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_write()
13864 	vcpu->run->mmio.is_write = 1;  in kvm_sev_es_mmio_write()
13865 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));  in kvm_sev_es_mmio_write()
13866 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_write()
13868 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_write()
13874 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,  in kvm_sev_es_mmio_read()  argument
13883 	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);  in kvm_sev_es_mmio_read()
13892 	frag = vcpu->mmio_fragments;  in kvm_sev_es_mmio_read()
13893 	vcpu->mmio_nr_fragments = 1;  in kvm_sev_es_mmio_read()
13898 	vcpu->mmio_needed = 1;  in kvm_sev_es_mmio_read()
13899 	vcpu->mmio_cur_fragment = 0;  in kvm_sev_es_mmio_read()
13901 	vcpu->run->mmio.phys_addr = gpa;  in kvm_sev_es_mmio_read()
13902 	vcpu->run->mmio.len = min(8u, frag->len);  in kvm_sev_es_mmio_read()
13903 	vcpu->run->mmio.is_write = 0;  in kvm_sev_es_mmio_read()
13904 	vcpu->run->exit_reason = KVM_EXIT_MMIO;  in kvm_sev_es_mmio_read()
13906 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;  in kvm_sev_es_mmio_read()
13912 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)  in advance_sev_es_emulated_pio()  argument
13914 	vcpu->arch.sev_pio_count -= count;  in advance_sev_es_emulated_pio()
13915 	vcpu->arch.sev_pio_data += count * size;  in advance_sev_es_emulated_pio()
13918 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13921 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_outs()  argument
13923 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_outs()
13924 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_outs()
13926 	vcpu->arch.pio.count = 0;  in complete_sev_es_emulated_outs()
13927 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_outs()
13928 		return kvm_sev_es_outs(vcpu, size, port);  in complete_sev_es_emulated_outs()
13932 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_outs()  argument
13937 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_outs()
13938 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);  in kvm_sev_es_outs()
13941 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_outs()
13946 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_outs()
13950 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;  in kvm_sev_es_outs()
13954 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13957 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)  in complete_sev_es_emulated_ins()  argument
13959 	unsigned count = vcpu->arch.pio.count;  in complete_sev_es_emulated_ins()
13960 	int size = vcpu->arch.pio.size;  in complete_sev_es_emulated_ins()
13961 	int port = vcpu->arch.pio.port;  in complete_sev_es_emulated_ins()
13963 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);  in complete_sev_es_emulated_ins()
13964 	advance_sev_es_emulated_pio(vcpu, count, size);  in complete_sev_es_emulated_ins()
13965 	if (vcpu->arch.sev_pio_count)  in complete_sev_es_emulated_ins()
13966 		return kvm_sev_es_ins(vcpu, size, port);  in complete_sev_es_emulated_ins()
13970 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_ins()  argument
13975 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);  in kvm_sev_es_ins()
13976 		if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))  in kvm_sev_es_ins()
13980 		advance_sev_es_emulated_pio(vcpu, count, size);  in kvm_sev_es_ins()
13981 		if (!vcpu->arch.sev_pio_count)  in kvm_sev_es_ins()
13985 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;  in kvm_sev_es_ins()
13989 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,  in kvm_sev_es_string_io()  argument
13993 	vcpu->arch.sev_pio_data = data;  in kvm_sev_es_string_io()
13994 	vcpu->arch.sev_pio_count = count;  in kvm_sev_es_string_io()
13995 	return in ? kvm_sev_es_ins(vcpu, size, port)  in kvm_sev_es_string_io()
13996 		  : kvm_sev_es_outs(vcpu, size, port);  in kvm_sev_es_string_io()