49 static u64 __compute_hcr(struct kvm_vcpu *vcpu)  in __compute_hcr()  argument
51 	u64 hcr = vcpu->arch.hcr_el2;  in __compute_hcr()
53 	if (!vcpu_has_nv(vcpu))  in __compute_hcr()
56 	if (is_hyp_ctxt(vcpu)) {  in __compute_hcr()
59 		if (!vcpu_el2_e2h_is_set(vcpu))  in __compute_hcr()
62 		write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);  in __compute_hcr()
65 	return hcr | (__vcpu_sys_reg(vcpu, HCR_EL2) & ~NV_HCR_GUEST_EXCLUDE);  in __compute_hcr()
68 static void __activate_cptr_traps(struct kvm_vcpu *vcpu)  in __activate_cptr_traps()  argument
84 		if (vcpu_has_sve(vcpu))  in __activate_cptr_traps()
87 		__activate_traps_fpsimd32(vcpu);  in __activate_cptr_traps()
90 	if (!vcpu_has_nv(vcpu))  in __activate_cptr_traps()
99 	if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu))  in __activate_cptr_traps()
106 	if (is_hyp_ctxt(vcpu))  in __activate_cptr_traps()
109 	cptr = vcpu_sanitised_cptr_el2(vcpu);  in __activate_cptr_traps()
130 	if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP))  in __activate_cptr_traps()
139 static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu)  in __deactivate_cptr_traps()  argument
149 static void __activate_traps(struct kvm_vcpu *vcpu)  in __activate_traps()  argument
153 	___activate_traps(vcpu, __compute_hcr(vcpu));  in __activate_traps()
158 		get_timer_map(vcpu, &map);  in __activate_traps()
164 		if (map.direct_ptimer == vcpu_ptimer(vcpu))  in __activate_traps()
165 			val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);  in __activate_traps()
166 		if (map.direct_ptimer == vcpu_hptimer(vcpu))  in __activate_traps()
167 			val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2);  in __activate_traps()
175 	__activate_cptr_traps(vcpu);  in __activate_traps()
181 static void __deactivate_traps(struct kvm_vcpu *vcpu)  in __deactivate_traps()  argument
185 	___deactivate_traps(vcpu);  in __deactivate_traps()
193 		get_timer_map(vcpu, &map);  in __deactivate_traps()
200 		if (map.direct_ptimer == vcpu_ptimer(vcpu))  in __deactivate_traps()
201 			__vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = val;  in __deactivate_traps()
202 		if (map.direct_ptimer == vcpu_hptimer(vcpu))  in __deactivate_traps()
203 			__vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2) = val;  in __deactivate_traps()
220 	__deactivate_cptr_traps(vcpu);  in __deactivate_traps()
234 static void __vcpu_load_activate_traps(struct kvm_vcpu *vcpu)  in __vcpu_load_activate_traps()  argument
239 	__activate_traps_common(vcpu);  in __vcpu_load_activate_traps()
243 static void __vcpu_put_deactivate_traps(struct kvm_vcpu *vcpu)  in __vcpu_put_deactivate_traps()  argument
248 	__deactivate_traps_common(vcpu);  in __vcpu_put_deactivate_traps()
252 void kvm_vcpu_load_vhe(struct kvm_vcpu *vcpu)  in kvm_vcpu_load_vhe()  argument
254 	host_data_ptr(host_ctxt)->__hyp_running_vcpu = vcpu;  in kvm_vcpu_load_vhe()
256 	__vcpu_load_switch_sysregs(vcpu);  in kvm_vcpu_load_vhe()
257 	__vcpu_load_activate_traps(vcpu);  in kvm_vcpu_load_vhe()
258 	__load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch);  in kvm_vcpu_load_vhe()
261 void kvm_vcpu_put_vhe(struct kvm_vcpu *vcpu)  in kvm_vcpu_put_vhe()  argument
263 	__vcpu_put_deactivate_traps(vcpu);  in kvm_vcpu_put_vhe()
264 	__vcpu_put_switch_sysregs(vcpu);  in kvm_vcpu_put_vhe()
269 static u64 compute_emulated_cntx_ctl_el0(struct kvm_vcpu *vcpu,  in compute_emulated_cntx_ctl_el0()  argument
278 		cval = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);  in compute_emulated_cntx_ctl_el0()
279 		ctl  = __vcpu_sys_reg(vcpu, CNTP_CTL_EL0);  in compute_emulated_cntx_ctl_el0()
280 		cnt  = compute_counter_value(vcpu_ptimer(vcpu));  in compute_emulated_cntx_ctl_el0()
283 		cval = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);  in compute_emulated_cntx_ctl_el0()
284 		ctl  = __vcpu_sys_reg(vcpu, CNTV_CTL_EL0);  in compute_emulated_cntx_ctl_el0()
285 		cnt  = compute_counter_value(vcpu_vtimer(vcpu));  in compute_emulated_cntx_ctl_el0()
297 static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code)  in kvm_hyp_handle_timer()  argument
307 	if (!is_hyp_ctxt(vcpu))  in kvm_hyp_handle_timer()
310 	esr = kvm_vcpu_get_esr(vcpu);  in kvm_hyp_handle_timer()
316 		val = compute_emulated_cntx_ctl_el0(vcpu, CNTP_CTL_EL0);  in kvm_hyp_handle_timer()
319 		if (vcpu_el2_e2h_is_set(vcpu))  in kvm_hyp_handle_timer()
322 			val = compute_emulated_cntx_ctl_el0(vcpu, CNTP_CTL_EL0);  in kvm_hyp_handle_timer()
325 		val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);  in kvm_hyp_handle_timer()
328 		if (vcpu_el2_e2h_is_set(vcpu)) {  in kvm_hyp_handle_timer()
332 				val -= timer_get_offset(vcpu_hptimer(vcpu));  in kvm_hyp_handle_timer()
334 			val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);  in kvm_hyp_handle_timer()
339 		val = compute_counter_value(vcpu_hptimer(vcpu));  in kvm_hyp_handle_timer()
342 		val = compute_emulated_cntx_ctl_el0(vcpu, CNTV_CTL_EL0);  in kvm_hyp_handle_timer()
345 		if (vcpu_el2_e2h_is_set(vcpu))  in kvm_hyp_handle_timer()
348 			val = compute_emulated_cntx_ctl_el0(vcpu, CNTV_CTL_EL0);  in kvm_hyp_handle_timer()
351 		val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);  in kvm_hyp_handle_timer()
354 		if (vcpu_el2_e2h_is_set(vcpu))  in kvm_hyp_handle_timer()
357 			val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);  in kvm_hyp_handle_timer()
361 		val = compute_counter_value(vcpu_hvtimer(vcpu));  in kvm_hyp_handle_timer()
367 	vcpu_set_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu), val);  in kvm_hyp_handle_timer()
368 	__kvm_skip_instr(vcpu);  in kvm_hyp_handle_timer()
373 static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code)  in kvm_hyp_handle_eret()  argument
375 	u64 esr = kvm_vcpu_get_esr(vcpu);  in kvm_hyp_handle_eret()
388 	if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) ||  in kvm_hyp_handle_eret()
389 	    (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_ERET))  in kvm_hyp_handle_eret()
397 		if (!(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)))  in kvm_hyp_handle_eret()
412 		if (!(vcpu_has_ptrauth(vcpu) && kvm_auth_eretax(vcpu, &elr)))  in kvm_hyp_handle_eret()
426 static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code)  in kvm_hyp_handle_tlbi_el2()  argument
448 	if (!(is_hyp_ctxt(vcpu)))  in kvm_hyp_handle_tlbi_el2()
451 	instr = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));  in kvm_hyp_handle_tlbi_el2()
452 	val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));  in kvm_hyp_handle_tlbi_el2()
454 	if ((kvm_supported_tlbi_s1e1_op(vcpu, instr) &&  in kvm_hyp_handle_tlbi_el2()
455 	     vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) ||  in kvm_hyp_handle_tlbi_el2()
456 	    kvm_supported_tlbi_s1e2_op (vcpu, instr))  in kvm_hyp_handle_tlbi_el2()
462 	__kvm_skip_instr(vcpu);  in kvm_hyp_handle_tlbi_el2()
467 static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code)  in kvm_hyp_handle_cpacr_el1()  argument
469 	u64 esr = kvm_vcpu_get_esr(vcpu);  in kvm_hyp_handle_cpacr_el1()
472 	if (!is_hyp_ctxt(vcpu) || esr_sys64_to_sysreg(esr) != SYS_CPACR_EL1)  in kvm_hyp_handle_cpacr_el1()
475 	rt = kvm_vcpu_sys_get_rt(vcpu);  in kvm_hyp_handle_cpacr_el1()
478 		vcpu_set_reg(vcpu, rt, __vcpu_sys_reg(vcpu, CPTR_EL2));  in kvm_hyp_handle_cpacr_el1()
480 		vcpu_write_sys_reg(vcpu, vcpu_get_reg(vcpu, rt), CPTR_EL2);  in kvm_hyp_handle_cpacr_el1()
481 		__activate_cptr_traps(vcpu);  in kvm_hyp_handle_cpacr_el1()
484 	__kvm_skip_instr(vcpu);  in kvm_hyp_handle_cpacr_el1()
489 static bool kvm_hyp_handle_zcr_el2(struct kvm_vcpu *vcpu, u64 *exit_code)  in kvm_hyp_handle_zcr_el2()  argument
491 	u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));  in kvm_hyp_handle_zcr_el2()
493 	if (!vcpu_has_nv(vcpu))  in kvm_hyp_handle_zcr_el2()
509 	kvm_hyp_handle_fpsimd(vcpu, exit_code);  in kvm_hyp_handle_zcr_el2()
513 static bool kvm_hyp_handle_sysreg_vhe(struct kvm_vcpu *vcpu, u64 *exit_code)  in kvm_hyp_handle_sysreg_vhe()  argument
515 	if (kvm_hyp_handle_tlbi_el2(vcpu, exit_code))  in kvm_hyp_handle_sysreg_vhe()
518 	if (kvm_hyp_handle_timer(vcpu, exit_code))  in kvm_hyp_handle_sysreg_vhe()
521 	if (kvm_hyp_handle_cpacr_el1(vcpu, exit_code))  in kvm_hyp_handle_sysreg_vhe()
524 	if (kvm_hyp_handle_zcr_el2(vcpu, exit_code))  in kvm_hyp_handle_sysreg_vhe()
527 	return kvm_hyp_handle_sysreg(vcpu, exit_code);  in kvm_hyp_handle_sysreg_vhe()
543 static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)  in fixup_guest_exit()  argument
545 	synchronize_vcpu_pstate(vcpu, exit_code);  in fixup_guest_exit()
551 	if (vcpu_has_nv(vcpu) && (read_sysreg(hcr_el2) & HCR_NV)) {  in fixup_guest_exit()
552 		u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);  in fixup_guest_exit()
563 		*vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT);  in fixup_guest_exit()
564 		*vcpu_cpsr(vcpu) |= mode;  in fixup_guest_exit()
567 	return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers);  in fixup_guest_exit()
571 static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)  in __kvm_vcpu_run_vhe()  argument
578 	guest_ctxt = &vcpu->arch.ctxt;  in __kvm_vcpu_run_vhe()
582 	fpsimd_lazy_switch_to_guest(vcpu);  in __kvm_vcpu_run_vhe()
590 	__activate_traps(vcpu);  in __kvm_vcpu_run_vhe()
592 	__kvm_adjust_pc(vcpu);  in __kvm_vcpu_run_vhe()
595 	__debug_switch_to_guest(vcpu);  in __kvm_vcpu_run_vhe()
599 		exit_code = __guest_enter(vcpu);  in __kvm_vcpu_run_vhe()
602 	} while (fixup_guest_exit(vcpu, &exit_code));  in __kvm_vcpu_run_vhe()
606 	__deactivate_traps(vcpu);  in __kvm_vcpu_run_vhe()
608 	fpsimd_lazy_switch_to_host(vcpu);  in __kvm_vcpu_run_vhe()
613 		__fpsimd_save_fpexc32(vcpu);  in __kvm_vcpu_run_vhe()
615 	__debug_switch_to_host(vcpu);  in __kvm_vcpu_run_vhe()
621 int __kvm_vcpu_run(struct kvm_vcpu *vcpu)  in __kvm_vcpu_run()  argument
638 	ret = __kvm_vcpu_run_vhe(vcpu);  in __kvm_vcpu_run()
658 	struct kvm_vcpu *vcpu;  in __hyp_call_panic()  local
661 	vcpu = host_ctxt->__hyp_running_vcpu;  in __hyp_call_panic()
663 	__deactivate_traps(vcpu);  in __hyp_call_panic()
669 	      read_sysreg(hpfar_el2), par, vcpu);  in __hyp_call_panic()