1 // SPDX-License-Identifier: GPL-2.0-only
16  * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
26  * Protect the per-CPU list with a per-CPU spinlock to handle task migration.
28  * ->sched_in() path will need to take the vCPU off the list of the _previous_
30  * occur if a wakeup IRQ arrives and attempts to acquire the lock.
36 	return &(to_vmx(vcpu)->pi_desc);  in vcpu_to_pi_desc()
47 	if (!try_cmpxchg64(&pi_desc->control, pold, new))  in pi_try_set_control()
48 		return -EBUSY;  in pi_try_set_control()
62 	 * To simplify hot-plug and dynamic toggling of APICv, keep PI.NDST and  in vmx_vcpu_pi_load()
63 	 * PI.SN up-to-date even if there is no assigned device or if APICv is  in vmx_vcpu_pi_load()
64 	 * deactivated due to a dynamic inhibit bit, e.g. for Hyper-V's SyncIC.  in vmx_vcpu_pi_load()
70 	 * If the vCPU wasn't on the wakeup list and wasn't migrated, then the  in vmx_vcpu_pi_load()
74 	if (pi_desc->nv != POSTED_INTR_WAKEUP_VECTOR && vcpu->cpu == cpu) {  in vmx_vcpu_pi_load()
87 	 * If the vCPU was waiting for wakeup, remove the vCPU from the wakeup  in vmx_vcpu_pi_load()
91 	if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR) {  in vmx_vcpu_pi_load()
92 		raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in vmx_vcpu_pi_load()
93 		list_del(&vmx->pi_wakeup_list);  in vmx_vcpu_pi_load()
94 		raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in vmx_vcpu_pi_load()
101 	old.control = READ_ONCE(pi_desc->control);  in vmx_vcpu_pi_load()
107 		 * handle task migration (@cpu != vcpu->cpu).  in vmx_vcpu_pi_load()
114 		 * descriptor was modified on "put" to use the wakeup vector.  in vmx_vcpu_pi_load()
124 	 * Clear SN before reading the bitmap.  The VT-d firmware  in vmx_vcpu_pi_load()
144  * WAKEUP as the notification vector in the PI descriptor.
155 	raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in pi_enable_wakeup_handler()
156 	list_add_tail(&vmx->pi_wakeup_list,  in pi_enable_wakeup_handler()
157 		      &per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu));  in pi_enable_wakeup_handler()
158 	raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));  in pi_enable_wakeup_handler()
162 	old.control = READ_ONCE(pi_desc->control);  in pi_enable_wakeup_handler()
164 		/* set 'NV' to 'wakeup vector' */  in pi_enable_wakeup_handler()
170 	 * Send a wakeup IPI to this CPU if an interrupt may have been posted  in pi_enable_wakeup_handler()
172 	 * will arrive on the non-wakeup vector.  An IPI is needed as calling  in pi_enable_wakeup_handler()
173 	 * try_to_wake_up() from ->sched_out() isn't allowed (IRQs are not  in pi_enable_wakeup_handler()
187 	 * invoked outside guest mode.   Return whether a blocked vCPU  in vmx_needs_pi_wakeup()
189 	 * using either IPI virtualization or VT-d PI, so that the  in vmx_needs_pi_wakeup()
193 	return vmx_can_use_ipiv(vcpu) || vmx_can_use_vtd_pi(vcpu->kvm);  in vmx_needs_pi_wakeup()
211 	if (vcpu->preempted)  in vmx_vcpu_pi_put()
228 		if (pi_test_on(&vmx->pi_desc))  in pi_wakeup_handler()
229 			kvm_vcpu_wake_up(&vmx->vcpu);  in pi_wakeup_handler()
252  * PI.NV to the wakeup vector, i.e. the assigned device
264  * vmx_pi_update_irte - set IRTE for Posted-Interrupts
285 	idx = srcu_read_lock(&kvm->irq_srcu);  in vmx_pi_update_irte()
286 	irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);  in vmx_pi_update_irte()
287 	if (guest_irq >= irq_rt->nr_rt_entries ||  in vmx_pi_update_irte()
288 	    hlist_empty(&irq_rt->map[guest_irq])) {  in vmx_pi_update_irte()
290 			     guest_irq, irq_rt->nr_rt_entries);  in vmx_pi_update_irte()
294 	hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {  in vmx_pi_update_irte()
295 		if (e->type != KVM_IRQ_ROUTING_MSI)  in vmx_pi_update_irte()
298 		 * VT-d PI cannot support posting multicast/broadcast  in vmx_pi_update_irte()
302 		 * For lowest-priority interrupts, we only support  in vmx_pi_update_irte()
305 		 * irqbalance to make the interrupts single-CPU.  in vmx_pi_update_irte()
307 		 * We will support full lowest-priority interrupt later.  in vmx_pi_update_irte()
317 			 * Make sure the IRTE is in remapped mode if  in vmx_pi_update_irte()
318 			 * we don't handle it in posted mode.  in vmx_pi_update_irte()
323 				   "failed to back to remapped mode, irq: %u\n",  in vmx_pi_update_irte()
334 		trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,  in vmx_pi_update_irte()
351 	srcu_read_unlock(&kvm->irq_srcu, idx);  in vmx_pi_update_irte()