Lines Matching full:apic
4 * Local APIC virtualization
63 * Enable local APIC timer advancement (tscdeadline mode only) with adaptive
79 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data);
80 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data);
87 static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) in kvm_lapic_set_reg() argument
89 __kvm_lapic_set_reg(apic->regs, reg_off, val); in kvm_lapic_set_reg()
98 static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) in kvm_lapic_get_reg64() argument
100 return __kvm_lapic_get_reg64(apic->regs, reg); in kvm_lapic_get_reg64()
109 static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, in kvm_lapic_set_reg64() argument
112 __kvm_lapic_set_reg64(apic->regs, reg, val); in kvm_lapic_set_reg64()
122 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_pending_eoi() local
124 return apic_test_vector(vector, apic->regs + APIC_ISR) || in kvm_apic_pending_eoi()
125 apic_test_vector(vector, apic->regs + APIC_IRR); in kvm_apic_pending_eoi()
144 static inline int apic_enabled(struct kvm_lapic *apic) in apic_enabled() argument
146 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic); in apic_enabled()
156 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) in kvm_x2apic_id() argument
158 return apic->vcpu->vcpu_id; in kvm_x2apic_id()
235 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_recalculate_phys_map() local
236 u32 x2apic_id = kvm_x2apic_id(apic); in kvm_recalculate_phys_map()
237 u32 xapic_id = kvm_xapic_id(apic); in kvm_recalculate_phys_map()
249 * Bail if a vCPU was added and/or enabled its APIC between allocating in kvm_recalculate_phys_map()
258 * Deliberately truncate the vCPU ID when detecting a mismatched APIC in kvm_recalculate_phys_map()
263 if (!apic_x2apic_mode(apic) && xapic_id != (u8)vcpu->vcpu_id) in kvm_recalculate_phys_map()
267 * Apply KVM's hotplug hack if userspace has enable 32-bit APIC IDs. in kvm_recalculate_phys_map()
274 * userspace has not enabled 32-bit x2APIC IDs. Each APIC is supposed in kvm_recalculate_phys_map()
276 * effective APIC ID, e.g. due to the x2APIC wrap or because the guest in kvm_recalculate_phys_map()
282 if (apic_x2apic_mode(apic) || x2apic_id > 0xff) in kvm_recalculate_phys_map()
283 new->phys_map[x2apic_id] = apic; in kvm_recalculate_phys_map()
285 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id]) in kvm_recalculate_phys_map()
286 new->phys_map[xapic_id] = apic; in kvm_recalculate_phys_map()
289 * Disable the optimized map if the physical APIC ID is already in kvm_recalculate_phys_map()
293 if (apic_x2apic_mode(apic)) in kvm_recalculate_phys_map()
301 new->phys_map[physical_id] = apic; in kvm_recalculate_phys_map()
310 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_recalculate_logical_map() local
319 if (!kvm_apic_sw_enabled(apic)) in kvm_recalculate_logical_map()
322 ldr = kvm_lapic_get_reg(apic, APIC_LDR); in kvm_recalculate_logical_map()
326 if (apic_x2apic_mode(apic)) { in kvm_recalculate_logical_map()
330 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT) in kvm_recalculate_logical_map()
354 if (apic_x2apic_mode(apic)) in kvm_recalculate_logical_map()
370 cluster[ldr] = apic; in kvm_recalculate_logical_map()
399 "Dirty APIC map without an in-kernel local APIC"); in kvm_recalculate_apic_map()
406 * or the APIC registers (if dirty). Note, on retry the map may have in kvm_recalculate_apic_map()
423 * with the highest x2APIC ID is toggling its APIC on and off. in kvm_recalculate_apic_map()
429 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic)); in kvm_recalculate_apic_map()
484 * Write kvm->arch.apic_map before clearing apic->apic_map_dirty. in kvm_recalculate_apic_map()
497 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) in apic_set_spiv() argument
501 kvm_lapic_set_reg(apic, APIC_SPIV, val); in apic_set_spiv()
503 if (enabled != apic->sw_enabled) { in apic_set_spiv()
504 apic->sw_enabled = enabled; in apic_set_spiv()
510 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in apic_set_spiv()
515 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu); in apic_set_spiv()
516 kvm_xen_sw_enable_lapic(apic->vcpu); in apic_set_spiv()
520 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id) in kvm_apic_set_xapic_id() argument
522 kvm_lapic_set_reg(apic, APIC_ID, id << 24); in kvm_apic_set_xapic_id()
523 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_xapic_id()
526 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id) in kvm_apic_set_ldr() argument
528 kvm_lapic_set_reg(apic, APIC_LDR, id); in kvm_apic_set_ldr()
529 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_ldr()
532 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val) in kvm_apic_set_dfr() argument
534 kvm_lapic_set_reg(apic, APIC_DFR, val); in kvm_apic_set_dfr()
535 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_dfr()
538 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id) in kvm_apic_set_x2apic_id() argument
542 WARN_ON_ONCE(id != apic->vcpu->vcpu_id); in kvm_apic_set_x2apic_id()
544 kvm_lapic_set_reg(apic, APIC_ID, id); in kvm_apic_set_x2apic_id()
545 kvm_lapic_set_reg(apic, APIC_LDR, ldr); in kvm_apic_set_x2apic_id()
546 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_x2apic_id()
549 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) in apic_lvt_enabled() argument
551 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); in apic_lvt_enabled()
554 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) in apic_lvtt_oneshot() argument
556 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT; in apic_lvtt_oneshot()
559 static inline int apic_lvtt_period(struct kvm_lapic *apic) in apic_lvtt_period() argument
561 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC; in apic_lvtt_period()
564 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) in apic_lvtt_tscdeadline() argument
566 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE; in apic_lvtt_tscdeadline()
574 static inline bool kvm_lapic_lvt_supported(struct kvm_lapic *apic, int lvt_index) in kvm_lapic_lvt_supported() argument
576 return apic->nr_lvt_entries > lvt_index; in kvm_lapic_lvt_supported()
586 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_version() local
592 v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); in kvm_apic_set_version()
604 kvm_lapic_set_reg(apic, APIC_LVR, v); in kvm_apic_set_version()
610 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_after_set_mcg_cap() local
613 if (!lapic_in_kernel(vcpu) || nr_lvt_entries == apic->nr_lvt_entries) in kvm_apic_after_set_mcg_cap()
617 for (i = apic->nr_lvt_entries; i < nr_lvt_entries; i++) in kvm_apic_after_set_mcg_cap()
618 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); in kvm_apic_after_set_mcg_cap()
620 apic->nr_lvt_entries = nr_lvt_entries; in kvm_apic_after_set_mcg_cap()
703 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_irr() local
704 bool irr_updated = __kvm_apic_update_irr(pir, apic->regs, max_irr); in kvm_apic_update_irr()
706 if (unlikely(!apic->apicv_active && irr_updated)) in kvm_apic_update_irr()
707 apic->irr_pending = true; in kvm_apic_update_irr()
712 static inline int apic_search_irr(struct kvm_lapic *apic) in apic_search_irr() argument
714 return find_highest_vector(apic->regs + APIC_IRR); in apic_search_irr()
717 static inline int apic_find_highest_irr(struct kvm_lapic *apic) in apic_find_highest_irr() argument
725 if (!apic->irr_pending) in apic_find_highest_irr()
728 result = apic_search_irr(apic); in apic_find_highest_irr()
734 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) in apic_clear_irr() argument
736 if (unlikely(apic->apicv_active)) { in apic_clear_irr()
737 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); in apic_clear_irr()
739 apic->irr_pending = false; in apic_clear_irr()
740 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); in apic_clear_irr()
741 if (apic_search_irr(apic) != -1) in apic_clear_irr()
742 apic->irr_pending = true; in apic_clear_irr()
748 apic_clear_irr(vec, vcpu->arch.apic); in kvm_apic_clear_irr()
752 static inline void apic_set_isr(int vec, struct kvm_lapic *apic) in apic_set_isr() argument
754 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) in apic_set_isr()
758 * With APIC virtualization enabled, all caching is disabled in apic_set_isr()
762 if (unlikely(apic->apicv_active)) in apic_set_isr()
763 kvm_x86_call(hwapic_isr_update)(apic->vcpu, vec); in apic_set_isr()
765 ++apic->isr_count; in apic_set_isr()
766 BUG_ON(apic->isr_count > MAX_APIC_VECTOR); in apic_set_isr()
772 apic->highest_isr_cache = vec; in apic_set_isr()
776 static inline int apic_find_highest_isr(struct kvm_lapic *apic) in apic_find_highest_isr() argument
782 * is always -1, with APIC virtualization enabled. in apic_find_highest_isr()
784 if (!apic->isr_count) in apic_find_highest_isr()
786 if (likely(apic->highest_isr_cache != -1)) in apic_find_highest_isr()
787 return apic->highest_isr_cache; in apic_find_highest_isr()
789 result = find_highest_vector(apic->regs + APIC_ISR); in apic_find_highest_isr()
795 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) in apic_clear_isr() argument
797 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) in apic_clear_isr()
801 * We do get here for APIC virtualization enabled if the guest in apic_clear_isr()
802 * uses the Hyper-V APIC enlightenment. In this case we may need in apic_clear_isr()
807 if (unlikely(apic->apicv_active)) in apic_clear_isr()
808 kvm_x86_call(hwapic_isr_update)(apic->vcpu, apic_find_highest_isr(apic)); in apic_clear_isr()
810 --apic->isr_count; in apic_clear_isr()
811 BUG_ON(apic->isr_count < 0); in apic_clear_isr()
812 apic->highest_isr_cache = -1; in apic_clear_isr()
818 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_hwapic_isr() local
820 if (WARN_ON_ONCE(!lapic_in_kernel(vcpu)) || !apic->apicv_active) in kvm_apic_update_hwapic_isr()
823 kvm_x86_call(hwapic_isr_update)(vcpu, apic_find_highest_isr(apic)); in kvm_apic_update_hwapic_isr()
834 return apic_find_highest_irr(vcpu->arch.apic); in kvm_lapic_find_highest_irr()
838 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
845 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_irq() local
847 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector, in kvm_apic_set_irq()
951 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) in apic_has_interrupt_for_ppr() argument
955 highest_irr = kvm_x86_call(sync_pir_to_irr)(apic->vcpu); in apic_has_interrupt_for_ppr()
957 highest_irr = apic_find_highest_irr(apic); in apic_has_interrupt_for_ppr()
963 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr) in __apic_update_ppr() argument
968 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI); in __apic_update_ppr()
969 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI); in __apic_update_ppr()
970 isr = apic_find_highest_isr(apic); in __apic_update_ppr()
980 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr); in __apic_update_ppr()
985 static void apic_update_ppr(struct kvm_lapic *apic) in apic_update_ppr() argument
989 if (__apic_update_ppr(apic, &ppr) && in apic_update_ppr()
990 apic_has_interrupt_for_ppr(apic, ppr) != -1) in apic_update_ppr()
991 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in apic_update_ppr()
996 apic_update_ppr(vcpu->arch.apic); in kvm_apic_update_ppr()
1000 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) in apic_set_tpr() argument
1002 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr); in apic_set_tpr()
1003 apic_update_ppr(apic); in apic_set_tpr()
1006 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda) in kvm_apic_broadcast() argument
1008 return mda == (apic_x2apic_mode(apic) ? in kvm_apic_broadcast()
1012 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda) in kvm_apic_match_physical_addr() argument
1014 if (kvm_apic_broadcast(apic, mda)) in kvm_apic_match_physical_addr()
1019 * were in x2APIC mode if the target APIC ID can't be encoded as an in kvm_apic_match_physical_addr()
1021 * start in xAPIC mode) with an APIC ID that is unaddressable in xAPIC in kvm_apic_match_physical_addr()
1022 * mode. Match the x2APIC ID if and only if the target APIC ID can't in kvm_apic_match_physical_addr()
1026 if (apic_x2apic_mode(apic) || mda > 0xff) in kvm_apic_match_physical_addr()
1027 return mda == kvm_x2apic_id(apic); in kvm_apic_match_physical_addr()
1029 return mda == kvm_xapic_id(apic); in kvm_apic_match_physical_addr()
1032 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda) in kvm_apic_match_logical_addr() argument
1036 if (kvm_apic_broadcast(apic, mda)) in kvm_apic_match_logical_addr()
1039 logical_id = kvm_lapic_get_reg(apic, APIC_LDR); in kvm_apic_match_logical_addr()
1041 if (apic_x2apic_mode(apic)) in kvm_apic_match_logical_addr()
1047 switch (kvm_lapic_get_reg(apic, APIC_DFR)) { in kvm_apic_match_logical_addr()
1058 /* The KVM local APIC implementation has two quirks:
1089 struct kvm_lapic *target = vcpu->arch.apic; in kvm_apic_match_dest()
1310 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, in __apic_accept_irq() argument
1315 struct kvm_vcpu *vcpu = apic->vcpu; in __apic_accept_irq()
1328 if (unlikely(!apic_enabled(apic))) in __apic_accept_irq()
1338 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { in __apic_accept_irq()
1341 apic->regs + APIC_TMR); in __apic_accept_irq()
1344 apic->regs + APIC_TMR); in __apic_accept_irq()
1347 kvm_x86_call(deliver_interrupt)(apic, delivery_mode, in __apic_accept_irq()
1375 apic->pending_events = (1UL << KVM_APIC_INIT); in __apic_accept_irq()
1383 apic->sipi_vector = vector; in __apic_accept_irq()
1386 set_bit(KVM_APIC_SIPI, &apic->pending_events); in __apic_accept_irq()
1456 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) in kvm_ioapic_handles_vector() argument
1458 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); in kvm_ioapic_handles_vector()
1461 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) in kvm_ioapic_send_eoi() argument
1466 if (!kvm_ioapic_handles_vector(apic, vector)) in kvm_ioapic_send_eoi()
1470 if (irqchip_split(apic->vcpu->kvm)) { in kvm_ioapic_send_eoi()
1471 apic->vcpu->arch.pending_ioapic_eoi = vector; in kvm_ioapic_send_eoi()
1472 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); in kvm_ioapic_send_eoi()
1476 if (apic_test_vector(vector, apic->regs + APIC_TMR)) in kvm_ioapic_send_eoi()
1481 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode); in kvm_ioapic_send_eoi()
1484 static int apic_set_eoi(struct kvm_lapic *apic) in apic_set_eoi() argument
1486 int vector = apic_find_highest_isr(apic); in apic_set_eoi()
1488 trace_kvm_eoi(apic, vector); in apic_set_eoi()
1497 apic_clear_isr(vector, apic); in apic_set_eoi()
1498 apic_update_ppr(apic); in apic_set_eoi()
1500 if (kvm_hv_synic_has_vector(apic->vcpu, vector)) in apic_set_eoi()
1501 kvm_hv_synic_send_eoi(apic->vcpu, vector); in apic_set_eoi()
1503 kvm_ioapic_send_eoi(apic, vector); in apic_set_eoi()
1504 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in apic_set_eoi()
1514 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_eoi_accelerated() local
1516 trace_kvm_eoi(apic, vector); in kvm_apic_set_eoi_accelerated()
1518 kvm_ioapic_send_eoi(apic, vector); in kvm_apic_set_eoi_accelerated()
1519 kvm_make_request(KVM_REQ_EVENT, apic->vcpu); in kvm_apic_set_eoi_accelerated()
1523 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high) in kvm_apic_send_ipi() argument
1537 if (apic_x2apic_mode(apic)) in kvm_apic_send_ipi()
1544 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL); in kvm_apic_send_ipi()
1548 static u32 apic_get_tmcct(struct kvm_lapic *apic) in apic_get_tmcct() argument
1553 ASSERT(apic != NULL); in apic_get_tmcct()
1556 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 || in apic_get_tmcct()
1557 apic->lapic_timer.period == 0) in apic_get_tmcct()
1561 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); in apic_get_tmcct()
1565 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period); in apic_get_tmcct()
1566 return div64_u64(ns, (apic->vcpu->kvm->arch.apic_bus_cycle_ns * in apic_get_tmcct()
1567 apic->divide_count)); in apic_get_tmcct()
1570 static void __report_tpr_access(struct kvm_lapic *apic, bool write) in __report_tpr_access() argument
1572 struct kvm_vcpu *vcpu = apic->vcpu; in __report_tpr_access()
1580 static inline void report_tpr_access(struct kvm_lapic *apic, bool write) in report_tpr_access() argument
1582 if (apic->vcpu->arch.tpr_access_reporting) in report_tpr_access()
1583 __report_tpr_access(apic, write); in report_tpr_access()
1586 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) in __apic_read() argument
1598 if (apic_lvtt_tscdeadline(apic)) in __apic_read()
1601 val = apic_get_tmcct(apic); in __apic_read()
1604 apic_update_ppr(apic); in __apic_read()
1605 val = kvm_lapic_get_reg(apic, offset); in __apic_read()
1608 report_tpr_access(apic, false); in __apic_read()
1611 val = kvm_lapic_get_reg(apic, offset); in __apic_read()
1627 u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic) in kvm_lapic_readable_reg_mask() argument
1652 if (kvm_lapic_lvt_supported(apic, LVT_CMCI)) in kvm_lapic_readable_reg_mask()
1656 if (!apic_x2apic_mode(apic)) in kvm_lapic_readable_reg_mask()
1665 static int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len, in kvm_lapic_reg_read() argument
1675 WARN_ON_ONCE(apic_x2apic_mode(apic) && offset == APIC_ICR); in kvm_lapic_reg_read()
1681 !(kvm_lapic_readable_reg_mask(apic) & APIC_REG_MASK(offset))) in kvm_lapic_reg_read()
1684 result = __apic_read(apic, offset & ~0xf); in kvm_lapic_reg_read()
1695 printk(KERN_ERR "Local APIC read with len = %x, " in kvm_lapic_reg_read()
1702 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) in apic_mmio_in_range() argument
1704 return addr >= apic->base_address && in apic_mmio_in_range()
1705 addr < apic->base_address + LAPIC_MMIO_LENGTH; in apic_mmio_in_range()
1711 struct kvm_lapic *apic = to_lapic(this); in apic_mmio_read() local
1712 u32 offset = address - apic->base_address; in apic_mmio_read()
1714 if (!apic_mmio_in_range(apic, address)) in apic_mmio_read()
1717 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { in apic_mmio_read()
1726 kvm_lapic_reg_read(apic, offset, len, data); in apic_mmio_read()
1731 static void update_divide_count(struct kvm_lapic *apic) in update_divide_count() argument
1735 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR); in update_divide_count()
1738 apic->divide_count = 0x1 << (tmp2 & 0x7); in update_divide_count()
1741 static void limit_periodic_timer_frequency(struct kvm_lapic *apic) in limit_periodic_timer_frequency() argument
1748 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { in limit_periodic_timer_frequency()
1751 if (apic->lapic_timer.period < min_period) { in limit_periodic_timer_frequency()
1755 apic->vcpu->vcpu_id, in limit_periodic_timer_frequency()
1756 apic->lapic_timer.period, min_period); in limit_periodic_timer_frequency()
1757 apic->lapic_timer.period = min_period; in limit_periodic_timer_frequency()
1762 static void cancel_hv_timer(struct kvm_lapic *apic);
1764 static void cancel_apic_timer(struct kvm_lapic *apic) in cancel_apic_timer() argument
1766 hrtimer_cancel(&apic->lapic_timer.timer); in cancel_apic_timer()
1768 if (apic->lapic_timer.hv_timer_in_use) in cancel_apic_timer()
1769 cancel_hv_timer(apic); in cancel_apic_timer()
1771 atomic_set(&apic->lapic_timer.pending, 0); in cancel_apic_timer()
1774 static void apic_update_lvtt(struct kvm_lapic *apic) in apic_update_lvtt() argument
1776 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & in apic_update_lvtt()
1777 apic->lapic_timer.timer_mode_mask; in apic_update_lvtt()
1779 if (apic->lapic_timer.timer_mode != timer_mode) { in apic_update_lvtt()
1780 if (apic_lvtt_tscdeadline(apic) != (timer_mode == in apic_update_lvtt()
1782 cancel_apic_timer(apic); in apic_update_lvtt()
1783 kvm_lapic_set_reg(apic, APIC_TMICT, 0); in apic_update_lvtt()
1784 apic->lapic_timer.period = 0; in apic_update_lvtt()
1785 apic->lapic_timer.tscdeadline = 0; in apic_update_lvtt()
1787 apic->lapic_timer.timer_mode = timer_mode; in apic_update_lvtt()
1788 limit_periodic_timer_frequency(apic); in apic_update_lvtt()
1799 struct kvm_lapic *apic = vcpu->arch.apic; in lapic_timer_int_injected() local
1800 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT); in lapic_timer_int_injected()
1802 if (kvm_apic_hw_enabled(apic)) { in lapic_timer_int_injected()
1804 void *bitmap = apic->regs + APIC_ISR; in lapic_timer_int_injected()
1806 if (apic->apicv_active) in lapic_timer_int_injected()
1807 bitmap = apic->regs + APIC_IRR; in lapic_timer_int_injected()
1817 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns; in __wait_lapic_expire()
1838 struct kvm_lapic *apic = vcpu->arch.apic; in adjust_lapic_timer_advance() local
1839 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns; in adjust_lapic_timer_advance()
1861 apic->lapic_timer.timer_advance_ns = timer_advance_ns; in adjust_lapic_timer_advance()
1866 struct kvm_lapic *apic = vcpu->arch.apic; in __kvm_wait_lapic_expire() local
1869 tsc_deadline = apic->lapic_timer.expired_tscdeadline; in __kvm_wait_lapic_expire()
1870 apic->lapic_timer.expired_tscdeadline = 0; in __kvm_wait_lapic_expire()
1890 vcpu->arch.apic->lapic_timer.expired_tscdeadline && in kvm_wait_lapic_expire()
1891 vcpu->arch.apic->lapic_timer.timer_advance_ns && in kvm_wait_lapic_expire()
1897 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic) in kvm_apic_inject_pending_timer_irqs() argument
1899 struct kvm_timer *ktimer = &apic->lapic_timer; in kvm_apic_inject_pending_timer_irqs()
1901 kvm_apic_local_deliver(apic, APIC_LVTT); in kvm_apic_inject_pending_timer_irqs()
1902 if (apic_lvtt_tscdeadline(apic)) { in kvm_apic_inject_pending_timer_irqs()
1904 } else if (apic_lvtt_oneshot(apic)) { in kvm_apic_inject_pending_timer_irqs()
1910 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) in apic_timer_expired() argument
1912 struct kvm_vcpu *vcpu = apic->vcpu; in apic_timer_expired()
1913 struct kvm_timer *ktimer = &apic->lapic_timer; in apic_timer_expired()
1915 if (atomic_read(&apic->lapic_timer.pending)) in apic_timer_expired()
1918 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) in apic_timer_expired()
1921 if (!from_timer_fn && apic->apicv_active) { in apic_timer_expired()
1923 kvm_apic_inject_pending_timer_irqs(apic); in apic_timer_expired()
1927 if (kvm_use_posted_timer_interrupt(apic->vcpu)) { in apic_timer_expired()
1935 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline && in apic_timer_expired()
1936 vcpu->arch.apic->lapic_timer.timer_advance_ns) in apic_timer_expired()
1938 kvm_apic_inject_pending_timer_irqs(apic); in apic_timer_expired()
1942 atomic_inc(&apic->lapic_timer.pending); in apic_timer_expired()
1948 static void start_sw_tscdeadline(struct kvm_lapic *apic) in start_sw_tscdeadline() argument
1950 struct kvm_timer *ktimer = &apic->lapic_timer; in start_sw_tscdeadline()
1954 struct kvm_vcpu *vcpu = apic->vcpu; in start_sw_tscdeadline()
1971 likely(ns > apic->lapic_timer.timer_advance_ns)) { in start_sw_tscdeadline()
1976 apic_timer_expired(apic, false); in start_sw_tscdeadline()
1981 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict) in tmict_to_ns() argument
1983 return (u64)tmict * apic->vcpu->kvm->arch.apic_bus_cycle_ns * in tmict_to_ns()
1984 (u64)apic->divide_count; in tmict_to_ns()
1987 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) in update_target_expiration() argument
1992 apic->lapic_timer.period = in update_target_expiration()
1993 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); in update_target_expiration()
1994 limit_periodic_timer_frequency(apic); in update_target_expiration()
1997 remaining = ktime_sub(apic->lapic_timer.target_expiration, now); in update_target_expiration()
2003 apic->divide_count, old_divisor); in update_target_expiration()
2005 apic->lapic_timer.tscdeadline += in update_target_expiration()
2006 nsec_to_cycles(apic->vcpu, ns_remaining_new) - in update_target_expiration()
2007 nsec_to_cycles(apic->vcpu, ns_remaining_old); in update_target_expiration()
2008 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); in update_target_expiration()
2011 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg) in set_target_expiration() argument
2018 apic->lapic_timer.period = in set_target_expiration()
2019 tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); in set_target_expiration()
2021 if (!apic->lapic_timer.period) { in set_target_expiration()
2022 apic->lapic_timer.tscdeadline = 0; in set_target_expiration()
2026 limit_periodic_timer_frequency(apic); in set_target_expiration()
2027 deadline = apic->lapic_timer.period; in set_target_expiration()
2029 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { in set_target_expiration()
2031 deadline = tmict_to_ns(apic, in set_target_expiration()
2032 kvm_lapic_get_reg(apic, count_reg)); in set_target_expiration()
2034 if (apic_lvtt_period(apic)) in set_target_expiration()
2035 deadline = apic->lapic_timer.period; in set_target_expiration()
2039 else if (unlikely(deadline > apic->lapic_timer.period)) { in set_target_expiration()
2044 apic->vcpu->vcpu_id, in set_target_expiration()
2046 kvm_lapic_get_reg(apic, count_reg), in set_target_expiration()
2047 deadline, apic->lapic_timer.period); in set_target_expiration()
2048 kvm_lapic_set_reg(apic, count_reg, 0); in set_target_expiration()
2049 deadline = apic->lapic_timer.period; in set_target_expiration()
2054 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + in set_target_expiration()
2055 nsec_to_cycles(apic->vcpu, deadline); in set_target_expiration()
2056 apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline); in set_target_expiration()
2061 static void advance_periodic_target_expiration(struct kvm_lapic *apic) in advance_periodic_target_expiration() argument
2074 apic->lapic_timer.target_expiration = in advance_periodic_target_expiration()
2075 ktime_add_ns(apic->lapic_timer.target_expiration, in advance_periodic_target_expiration()
2076 apic->lapic_timer.period); in advance_periodic_target_expiration()
2077 delta = ktime_sub(apic->lapic_timer.target_expiration, now); in advance_periodic_target_expiration()
2078 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + in advance_periodic_target_expiration()
2079 nsec_to_cycles(apic->vcpu, delta); in advance_periodic_target_expiration()
2082 static void start_sw_period(struct kvm_lapic *apic) in start_sw_period() argument
2084 if (!apic->lapic_timer.period) in start_sw_period()
2088 apic->lapic_timer.target_expiration)) { in start_sw_period()
2089 apic_timer_expired(apic, false); in start_sw_period()
2091 if (apic_lvtt_oneshot(apic)) in start_sw_period()
2094 advance_periodic_target_expiration(apic); in start_sw_period()
2097 hrtimer_start(&apic->lapic_timer.timer, in start_sw_period()
2098 apic->lapic_timer.target_expiration, in start_sw_period()
2107 return vcpu->arch.apic->lapic_timer.hv_timer_in_use; in kvm_lapic_hv_timer_in_use()
2110 static void cancel_hv_timer(struct kvm_lapic *apic) in cancel_hv_timer() argument
2113 WARN_ON(!apic->lapic_timer.hv_timer_in_use); in cancel_hv_timer()
2114 kvm_x86_call(cancel_hv_timer)(apic->vcpu); in cancel_hv_timer()
2115 apic->lapic_timer.hv_timer_in_use = false; in cancel_hv_timer()
2118 static bool start_hv_timer(struct kvm_lapic *apic) in start_hv_timer() argument
2120 struct kvm_timer *ktimer = &apic->lapic_timer; in start_hv_timer()
2121 struct kvm_vcpu *vcpu = apic->vcpu; in start_hv_timer()
2142 if (!apic_lvtt_period(apic)) { in start_hv_timer()
2148 cancel_hv_timer(apic); in start_hv_timer()
2150 apic_timer_expired(apic, false); in start_hv_timer()
2151 cancel_hv_timer(apic); in start_hv_timer()
2160 static void start_sw_timer(struct kvm_lapic *apic) in start_sw_timer() argument
2162 struct kvm_timer *ktimer = &apic->lapic_timer; in start_sw_timer()
2165 if (apic->lapic_timer.hv_timer_in_use) in start_sw_timer()
2166 cancel_hv_timer(apic); in start_sw_timer()
2167 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending)) in start_sw_timer()
2170 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) in start_sw_timer()
2171 start_sw_period(apic); in start_sw_timer()
2172 else if (apic_lvtt_tscdeadline(apic)) in start_sw_timer()
2173 start_sw_tscdeadline(apic); in start_sw_timer()
2174 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); in start_sw_timer()
2177 static void restart_apic_timer(struct kvm_lapic *apic) in restart_apic_timer() argument
2181 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending)) in restart_apic_timer()
2184 if (!start_hv_timer(apic)) in restart_apic_timer()
2185 start_sw_timer(apic); in restart_apic_timer()
2192 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_expired_hv_timer() local
2196 if (!apic->lapic_timer.hv_timer_in_use) in kvm_lapic_expired_hv_timer()
2199 apic_timer_expired(apic, false); in kvm_lapic_expired_hv_timer()
2200 cancel_hv_timer(apic); in kvm_lapic_expired_hv_timer()
2202 if (apic_lvtt_period(apic) && apic->lapic_timer.period) { in kvm_lapic_expired_hv_timer()
2203 advance_periodic_target_expiration(apic); in kvm_lapic_expired_hv_timer()
2204 restart_apic_timer(apic); in kvm_lapic_expired_hv_timer()
2213 restart_apic_timer(vcpu->arch.apic); in kvm_lapic_switch_to_hv_timer()
2218 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_switch_to_sw_timer() local
2222 if (apic->lapic_timer.hv_timer_in_use) in kvm_lapic_switch_to_sw_timer()
2223 start_sw_timer(apic); in kvm_lapic_switch_to_sw_timer()
2229 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_restart_hv_timer() local
2231 WARN_ON(!apic->lapic_timer.hv_timer_in_use); in kvm_lapic_restart_hv_timer()
2232 restart_apic_timer(apic); in kvm_lapic_restart_hv_timer()
2235 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg) in __start_apic_timer() argument
2237 atomic_set(&apic->lapic_timer.pending, 0); in __start_apic_timer()
2239 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) in __start_apic_timer()
2240 && !set_target_expiration(apic, count_reg)) in __start_apic_timer()
2243 restart_apic_timer(apic); in __start_apic_timer()
2246 static void start_apic_timer(struct kvm_lapic *apic) in start_apic_timer() argument
2248 __start_apic_timer(apic, APIC_TMICT); in start_apic_timer()
2251 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) in apic_manage_nmi_watchdog() argument
2255 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) { in apic_manage_nmi_watchdog()
2256 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode; in apic_manage_nmi_watchdog()
2258 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); in apic_manage_nmi_watchdog()
2260 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode); in apic_manage_nmi_watchdog()
2274 static int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) in kvm_lapic_reg_write() argument
2281 case APIC_ID: /* Local APIC ID */ in kvm_lapic_reg_write()
2282 if (!apic_x2apic_mode(apic)) { in kvm_lapic_reg_write()
2283 kvm_apic_set_xapic_id(apic, val >> 24); in kvm_lapic_reg_write()
2290 report_tpr_access(apic, true); in kvm_lapic_reg_write()
2291 apic_set_tpr(apic, val & 0xff); in kvm_lapic_reg_write()
2295 apic_set_eoi(apic); in kvm_lapic_reg_write()
2299 if (!apic_x2apic_mode(apic)) in kvm_lapic_reg_write()
2300 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK); in kvm_lapic_reg_write()
2306 if (!apic_x2apic_mode(apic)) in kvm_lapic_reg_write()
2307 kvm_apic_set_dfr(apic, val | 0x0FFFFFFF); in kvm_lapic_reg_write()
2314 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) in kvm_lapic_reg_write()
2316 apic_set_spiv(apic, val & mask); in kvm_lapic_reg_write()
2320 for (i = 0; i < apic->nr_lvt_entries; i++) { in kvm_lapic_reg_write()
2321 kvm_lapic_set_reg(apic, APIC_LVTx(i), in kvm_lapic_reg_write()
2322 kvm_lapic_get_reg(apic, APIC_LVTx(i)) | APIC_LVT_MASKED); in kvm_lapic_reg_write()
2324 apic_update_lvtt(apic); in kvm_lapic_reg_write()
2325 atomic_set(&apic->lapic_timer.pending, 0); in kvm_lapic_reg_write()
2331 WARN_ON_ONCE(apic_x2apic_mode(apic)); in kvm_lapic_reg_write()
2335 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2)); in kvm_lapic_reg_write()
2336 kvm_lapic_set_reg(apic, APIC_ICR, val); in kvm_lapic_reg_write()
2339 if (apic_x2apic_mode(apic)) in kvm_lapic_reg_write()
2342 kvm_lapic_set_reg(apic, APIC_ICR2, val & 0xff000000); in kvm_lapic_reg_write()
2346 apic_manage_nmi_watchdog(apic, val); in kvm_lapic_reg_write()
2354 if (!kvm_lapic_lvt_supported(apic, index)) { in kvm_lapic_reg_write()
2358 if (!kvm_apic_sw_enabled(apic)) in kvm_lapic_reg_write()
2361 kvm_lapic_set_reg(apic, reg, val); in kvm_lapic_reg_write()
2366 if (!kvm_apic_sw_enabled(apic)) in kvm_lapic_reg_write()
2368 val &= (apic_lvt_mask[LVT_TIMER] | apic->lapic_timer.timer_mode_mask); in kvm_lapic_reg_write()
2369 kvm_lapic_set_reg(apic, APIC_LVTT, val); in kvm_lapic_reg_write()
2370 apic_update_lvtt(apic); in kvm_lapic_reg_write()
2374 if (apic_lvtt_tscdeadline(apic)) in kvm_lapic_reg_write()
2377 cancel_apic_timer(apic); in kvm_lapic_reg_write()
2378 kvm_lapic_set_reg(apic, APIC_TMICT, val); in kvm_lapic_reg_write()
2379 start_apic_timer(apic); in kvm_lapic_reg_write()
2383 uint32_t old_divisor = apic->divide_count; in kvm_lapic_reg_write()
2385 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb); in kvm_lapic_reg_write()
2386 update_divide_count(apic); in kvm_lapic_reg_write()
2387 if (apic->divide_count != old_divisor && in kvm_lapic_reg_write()
2388 apic->lapic_timer.period) { in kvm_lapic_reg_write()
2389 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reg_write()
2390 update_target_expiration(apic, old_divisor); in kvm_lapic_reg_write()
2391 restart_apic_timer(apic); in kvm_lapic_reg_write()
2396 if (apic_x2apic_mode(apic) && val != 0) in kvm_lapic_reg_write()
2405 if (!apic_x2apic_mode(apic) || (val & ~APIC_VECTOR_MASK)) in kvm_lapic_reg_write()
2408 kvm_apic_send_ipi(apic, APIC_DEST_SELF | val, 0); in kvm_lapic_reg_write()
2416 * Recalculate APIC maps if necessary, e.g. if the software enable bit in kvm_lapic_reg_write()
2417 * was toggled, the APIC ID changed, etc... The maps are marked dirty in kvm_lapic_reg_write()
2420 kvm_recalculate_apic_map(apic->vcpu->kvm); in kvm_lapic_reg_write()
2428 struct kvm_lapic *apic = to_lapic(this); in apic_mmio_write() local
2429 unsigned int offset = address - apic->base_address; in apic_mmio_write()
2432 if (!apic_mmio_in_range(apic, address)) in apic_mmio_write()
2435 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) { in apic_mmio_write()
2444 * APIC register must be aligned on 128-bits boundary. in apic_mmio_write()
2453 kvm_lapic_reg_write(apic, offset & 0xff0, val); in apic_mmio_write()
2460 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0); in kvm_lapic_set_eoi()
2466 int kvm_x2apic_icr_write(struct kvm_lapic *apic, u64 data) in kvm_x2apic_icr_write() argument
2481 kvm_apic_send_ipi(apic, (u32)data, (u32)(data >> 32)); in kvm_x2apic_icr_write()
2483 kvm_lapic_set_reg(apic, APIC_ICR, data); in kvm_x2apic_icr_write()
2484 kvm_lapic_set_reg(apic, APIC_ICR2, data >> 32); in kvm_x2apic_icr_write()
2486 kvm_lapic_set_reg64(apic, APIC_ICR, data); in kvm_x2apic_icr_write()
2492 static u64 kvm_x2apic_icr_read(struct kvm_lapic *apic) in kvm_x2apic_icr_read() argument
2495 return (u64)kvm_lapic_get_reg(apic, APIC_ICR) | in kvm_x2apic_icr_read()
2496 (u64)kvm_lapic_get_reg(apic, APIC_ICR2) << 32; in kvm_x2apic_icr_read()
2498 return kvm_lapic_get_reg64(apic, APIC_ICR); in kvm_x2apic_icr_read()
2501 /* emulate APIC access in a trap manner */
2504 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_write_nodecode() local
2512 * virtual APIC state, but KVM needs to conditionally modify the value in kvm_apic_write_nodecode()
2517 if (apic_x2apic_mode(apic) && offset == APIC_ICR) in kvm_apic_write_nodecode()
2518 WARN_ON_ONCE(kvm_x2apic_icr_write(apic, kvm_x2apic_icr_read(apic))); in kvm_apic_write_nodecode()
2520 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); in kvm_apic_write_nodecode()
2526 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_free_lapic() local
2528 if (!vcpu->arch.apic) { in kvm_free_lapic()
2533 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_free_lapic()
2538 if (!apic->sw_enabled) in kvm_free_lapic()
2541 if (apic->regs) in kvm_free_lapic()
2542 free_page((unsigned long)apic->regs); in kvm_free_lapic()
2544 kfree(apic); in kvm_free_lapic()
2554 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_get_lapic_tscdeadline_msr() local
2556 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) in kvm_get_lapic_tscdeadline_msr()
2559 return apic->lapic_timer.tscdeadline; in kvm_get_lapic_tscdeadline_msr()
2564 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_set_lapic_tscdeadline_msr() local
2566 if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic)) in kvm_set_lapic_tscdeadline_msr()
2569 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_set_lapic_tscdeadline_msr()
2570 apic->lapic_timer.tscdeadline = data; in kvm_set_lapic_tscdeadline_msr()
2571 start_apic_timer(apic); in kvm_set_lapic_tscdeadline_msr()
2576 apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4); in kvm_lapic_set_tpr()
2583 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI); in kvm_lapic_get_cr8()
2591 struct kvm_lapic *apic = vcpu->arch.apic; in __kvm_apic_set_base() local
2598 if (!apic) in __kvm_apic_set_base()
2604 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in __kvm_apic_set_base()
2610 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in __kvm_apic_set_base()
2616 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id); in __kvm_apic_set_base()
2618 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in __kvm_apic_set_base()
2626 apic->base_address = apic->vcpu->arch.apic_base & in __kvm_apic_set_base()
2630 apic->base_address != APIC_DEFAULT_PHYS_BASE) { in __kvm_apic_set_base()
2631 kvm_set_apicv_inhibit(apic->vcpu->kvm, in __kvm_apic_set_base()
2663 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_update_apicv() local
2678 apic->irr_pending = true; in kvm_apic_update_apicv()
2680 if (apic->apicv_active) in kvm_apic_update_apicv()
2681 apic->isr_count = 1; in kvm_apic_update_apicv()
2683 apic->isr_count = count_vectors(apic->regs + APIC_ISR); in kvm_apic_update_apicv()
2685 apic->highest_isr_cache = -1; in kvm_apic_update_apicv()
2749 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_reset() local
2762 * optimized APIC map if some other task has dirtied the map. in kvm_lapic_reset()
2764 * all APIC state has been initialized (see below). in kvm_lapic_reset()
2769 if (!apic) in kvm_lapic_reset()
2772 /* Stop the timer in case it's a reset to an active apic */ in kvm_lapic_reset()
2773 hrtimer_cancel(&apic->lapic_timer.timer); in kvm_lapic_reset()
2775 /* The xAPIC ID is set at RESET even if the APIC was already enabled. */ in kvm_lapic_reset()
2777 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id); in kvm_lapic_reset()
2778 kvm_apic_set_version(apic->vcpu); in kvm_lapic_reset()
2780 for (i = 0; i < apic->nr_lvt_entries; i++) in kvm_lapic_reset()
2781 kvm_lapic_set_reg(apic, APIC_LVTx(i), APIC_LVT_MASKED); in kvm_lapic_reset()
2782 apic_update_lvtt(apic); in kvm_lapic_reset()
2785 kvm_lapic_set_reg(apic, APIC_LVT0, in kvm_lapic_reset()
2787 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); in kvm_lapic_reset()
2789 kvm_apic_set_dfr(apic, 0xffffffffU); in kvm_lapic_reset()
2790 apic_set_spiv(apic, 0xff); in kvm_lapic_reset()
2791 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0); in kvm_lapic_reset()
2792 if (!apic_x2apic_mode(apic)) in kvm_lapic_reset()
2793 kvm_apic_set_ldr(apic, 0); in kvm_lapic_reset()
2794 kvm_lapic_set_reg(apic, APIC_ESR, 0); in kvm_lapic_reset()
2795 if (!apic_x2apic_mode(apic)) { in kvm_lapic_reset()
2796 kvm_lapic_set_reg(apic, APIC_ICR, 0); in kvm_lapic_reset()
2797 kvm_lapic_set_reg(apic, APIC_ICR2, 0); in kvm_lapic_reset()
2799 kvm_lapic_set_reg64(apic, APIC_ICR, 0); in kvm_lapic_reset()
2801 kvm_lapic_set_reg(apic, APIC_TDCR, 0); in kvm_lapic_reset()
2802 kvm_lapic_set_reg(apic, APIC_TMICT, 0); in kvm_lapic_reset()
2804 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0); in kvm_lapic_reset()
2805 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0); in kvm_lapic_reset()
2806 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0); in kvm_lapic_reset()
2809 update_divide_count(apic); in kvm_lapic_reset()
2810 atomic_set(&apic->lapic_timer.pending, 0); in kvm_lapic_reset()
2813 apic_update_ppr(apic); in kvm_lapic_reset()
2814 if (apic->apicv_active) { in kvm_lapic_reset()
2831 static bool lapic_is_periodic(struct kvm_lapic *apic) in lapic_is_periodic() argument
2833 return apic_lvtt_period(apic); in lapic_is_periodic()
2838 struct kvm_lapic *apic = vcpu->arch.apic; in apic_has_pending_timer() local
2840 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT)) in apic_has_pending_timer()
2841 return atomic_read(&apic->lapic_timer.pending); in apic_has_pending_timer()
2846 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) in kvm_apic_local_deliver() argument
2848 u32 reg = kvm_lapic_get_reg(apic, lvt_type); in kvm_apic_local_deliver()
2852 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { in kvm_apic_local_deliver()
2857 r = __apic_accept_irq(apic, mode, vector, 1, trig_mode, NULL); in kvm_apic_local_deliver()
2859 guest_cpuid_is_intel_compatible(apic->vcpu)) in kvm_apic_local_deliver()
2860 kvm_lapic_set_reg(apic, APIC_LVTPC, reg | APIC_LVT_MASKED); in kvm_apic_local_deliver()
2868 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_nmi_wd_deliver() local
2870 if (apic) in kvm_apic_nmi_wd_deliver()
2871 kvm_apic_local_deliver(apic, APIC_LVT0); in kvm_apic_nmi_wd_deliver()
2882 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); in apic_timer_fn() local
2884 apic_timer_expired(apic, true); in apic_timer_fn()
2886 if (lapic_is_periodic(apic)) { in apic_timer_fn()
2887 advance_periodic_target_expiration(apic); in apic_timer_fn()
2896 struct kvm_lapic *apic; in kvm_create_lapic() local
2905 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT); in kvm_create_lapic()
2906 if (!apic) in kvm_create_lapic()
2909 vcpu->arch.apic = apic; in kvm_create_lapic()
2912 apic->regs = kvm_x86_call(alloc_apic_backing_page)(vcpu); in kvm_create_lapic()
2914 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); in kvm_create_lapic()
2915 if (!apic->regs) { in kvm_create_lapic()
2916 printk(KERN_ERR "malloc apic regs error for vcpu %x\n", in kvm_create_lapic()
2920 apic->vcpu = vcpu; in kvm_create_lapic()
2922 apic->nr_lvt_entries = kvm_apic_calc_nr_lvt_entries(vcpu); in kvm_create_lapic()
2924 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, in kvm_create_lapic()
2926 apic->lapic_timer.timer.function = apic_timer_fn; in kvm_create_lapic()
2928 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT; in kvm_create_lapic()
2931 * Stuff the APIC ENABLE bit in lieu of temporarily incrementing in kvm_create_lapic()
2936 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); in kvm_create_lapic()
2949 apic->apicv_active = true; in kvm_create_lapic()
2955 kfree(apic); in kvm_create_lapic()
2956 vcpu->arch.apic = NULL; in kvm_create_lapic()
2963 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_has_interrupt() local
2969 __apic_update_ppr(apic, &ppr); in kvm_apic_has_interrupt()
2970 return apic_has_interrupt_for_ppr(apic, ppr); in kvm_apic_has_interrupt()
2976 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0); in kvm_apic_accept_pic_intr()
2978 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) in kvm_apic_accept_pic_intr()
2988 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_inject_apic_timer_irqs() local
2990 if (atomic_read(&apic->lapic_timer.pending) > 0) { in kvm_inject_apic_timer_irqs()
2991 kvm_apic_inject_pending_timer_irqs(apic); in kvm_inject_apic_timer_irqs()
2992 atomic_set(&apic->lapic_timer.pending, 0); in kvm_inject_apic_timer_irqs()
2998 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_ack_interrupt() local
3001 if (WARN_ON_ONCE(vector < 0 || !apic)) in kvm_apic_ack_interrupt()
3005 * We get here even with APIC virtualization enabled, if doing in kvm_apic_ack_interrupt()
3011 apic_clear_irr(vector, apic); in kvm_apic_ack_interrupt()
3018 apic_update_ppr(apic); in kvm_apic_ack_interrupt()
3026 apic_set_isr(vector, apic); in kvm_apic_ack_interrupt()
3027 __apic_update_ppr(apic, &ppr); in kvm_apic_ack_interrupt()
3036 if (apic_x2apic_mode(vcpu->arch.apic)) { in kvm_apic_state_fixup()
3037 u32 x2apic_id = kvm_x2apic_id(vcpu->arch.apic); in kvm_apic_state_fixup()
3047 * Ignore the userspace value when setting APIC state. in kvm_apic_state_fixup()
3087 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); in kvm_apic_get_state()
3094 __apic_read(vcpu->arch.apic, APIC_TMCCT)); in kvm_apic_get_state()
3101 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_set_state() local
3107 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV))); in kvm_apic_set_state()
3114 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s)); in kvm_apic_set_state()
3116 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY); in kvm_apic_set_state()
3120 apic_update_ppr(apic); in kvm_apic_set_state()
3121 cancel_apic_timer(apic); in kvm_apic_set_state()
3122 apic->lapic_timer.expired_tscdeadline = 0; in kvm_apic_set_state()
3123 apic_update_lvtt(apic); in kvm_apic_set_state()
3124 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); in kvm_apic_set_state()
3125 update_divide_count(apic); in kvm_apic_set_state()
3126 __start_apic_timer(apic, APIC_TMCCT); in kvm_apic_set_state()
3127 kvm_lapic_set_reg(apic, APIC_TMCCT, 0); in kvm_apic_set_state()
3129 if (apic->apicv_active) { in kvm_apic_set_state()
3131 kvm_x86_call(hwapic_isr_update)(vcpu, apic_find_highest_isr(apic)); in kvm_apic_set_state()
3150 timer = &vcpu->arch.apic->lapic_timer.timer; in __kvm_migrate_apic_timer()
3163 struct kvm_lapic *apic) in apic_sync_pv_eoi_from_guest() argument
3181 vector = apic_set_eoi(apic); in apic_sync_pv_eoi_from_guest()
3182 trace_kvm_pv_eoi(apic, vector); in apic_sync_pv_eoi_from_guest()
3190 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic); in kvm_lapic_sync_from_vapic()
3195 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, in kvm_lapic_sync_from_vapic()
3199 apic_set_tpr(vcpu->arch.apic, data & 0xff); in kvm_lapic_sync_from_vapic()
3209 struct kvm_lapic *apic) in apic_sync_pv_eoi_to_guest() argument
3213 apic->irr_pending || in apic_sync_pv_eoi_to_guest()
3215 apic->highest_isr_cache == -1 || in apic_sync_pv_eoi_to_guest()
3217 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) { in apic_sync_pv_eoi_to_guest()
3225 pv_eoi_set_pending(apic->vcpu); in apic_sync_pv_eoi_to_guest()
3232 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_lapic_sync_to_vapic() local
3234 apic_sync_pv_eoi_to_guest(vcpu, apic); in kvm_lapic_sync_to_vapic()
3239 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff; in kvm_lapic_sync_to_vapic()
3240 max_irr = apic_find_highest_irr(apic); in kvm_lapic_sync_to_vapic()
3243 max_isr = apic_find_highest_isr(apic); in kvm_lapic_sync_to_vapic()
3248 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data, in kvm_lapic_sync_to_vapic()
3256 &vcpu->arch.apic->vapic_cache, in kvm_lapic_set_vapic_addr()
3264 vcpu->arch.apic->vapic_addr = vapic_addr; in kvm_lapic_set_vapic_addr()
3268 static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data) in kvm_lapic_msr_read() argument
3273 *data = kvm_x2apic_icr_read(apic); in kvm_lapic_msr_read()
3277 if (kvm_lapic_reg_read(apic, reg, 4, &low)) in kvm_lapic_msr_read()
3285 static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data) in kvm_lapic_msr_write() argument
3293 return kvm_x2apic_icr_write(apic, data); in kvm_lapic_msr_write()
3299 return kvm_lapic_reg_write(apic, reg, (u32)data); in kvm_lapic_msr_write()
3304 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_x2apic_msr_write() local
3307 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) in kvm_x2apic_msr_write()
3310 return kvm_lapic_msr_write(apic, reg, data); in kvm_x2apic_msr_write()
3315 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_x2apic_msr_read() local
3318 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic)) in kvm_x2apic_msr_read()
3321 return kvm_lapic_msr_read(apic, reg, data); in kvm_x2apic_msr_read()
3329 return kvm_lapic_msr_write(vcpu->arch.apic, reg, data); in kvm_hv_vapic_msr_write()
3337 return kvm_lapic_msr_read(vcpu->arch.apic, reg, data); in kvm_hv_vapic_msr_read()
3368 struct kvm_lapic *apic = vcpu->arch.apic; in kvm_apic_accept_events() local
3393 clear_bit(KVM_APIC_SIPI, &apic->pending_events); in kvm_apic_accept_events()
3397 if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) { in kvm_apic_accept_events()
3399 if (kvm_vcpu_is_bsp(apic->vcpu)) in kvm_apic_accept_events()
3404 if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) { in kvm_apic_accept_events()
3408 sipi_vector = apic->sipi_vector; in kvm_apic_accept_events()