29 #define IMR(offset)				XE_REG(offset + 0x4)
30 #define IIR(offset)				XE_REG(offset + 0x8)
31 #define IER(offset)				XE_REG(offset + 0xc)
33 static int xe_irq_msix_init(struct xe_device *xe);
34 static void xe_irq_msix_free(struct xe_device *xe);
35 static int xe_irq_msix_request_irqs(struct xe_device *xe);
36 static void xe_irq_msix_synchronize_irq(struct xe_device *xe);
42 	if (val == 0)  in assert_iir_is_zero()
45 	drm_WARN(&mmio->tile->xe->drm, 1,  in assert_iir_is_zero()
46 		 "Interrupt register 0x%x is not zero: 0x%08x\n",  in assert_iir_is_zero()
48 	xe_mmio_write32(mmio, reg, 0xffffffff);  in assert_iir_is_zero()
50 	xe_mmio_write32(mmio, reg, 0xffffffff);  in assert_iir_is_zero()
80 	xe_mmio_write32(mmio, IMR(irqregs), ~0);  in mask_and_disable()
84 	xe_mmio_write32(mmio, IER(irqregs), 0);  in mask_and_disable()
87 	xe_mmio_write32(mmio, IIR(irqregs), ~0);  in mask_and_disable()
89 	xe_mmio_write32(mmio, IIR(irqregs), ~0);  in mask_and_disable()
93 static u32 xelp_intr_disable(struct xe_device *xe)  in xelp_intr_disable()  argument
95 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in xelp_intr_disable()
97 	xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0);  in xelp_intr_disable()
109 gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl)  in gu_misc_irq_ack()  argument
111 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in gu_misc_irq_ack()
115 		return 0;  in gu_misc_irq_ack()
124 static inline void xelp_intr_enable(struct xe_device *xe, bool stall)  in xelp_intr_enable()  argument
126 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in xelp_intr_enable()
136 	struct xe_device *xe = gt_to_xe(gt);  in xe_irq_enable_hwe()  local
140 	u32 gsc_mask = 0;  in xe_irq_enable_hwe()
141 	u32 heci_mask = 0;  in xe_irq_enable_hwe()
143 	if (xe_device_uses_memirq(xe))  in xe_irq_enable_hwe()
146 	if (xe_device_uc_enabled(xe)) {  in xe_irq_enable_hwe()
179 		if (ccs_mask & (BIT(0)|BIT(1)))  in xe_irq_enable_hwe()
185 	if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) {  in xe_irq_enable_hwe()
201 		} else if (xe->info.has_heci_gscfi) {  in xe_irq_enable_hwe()
215 gt_engine_identity(struct xe_device *xe,  in gt_engine_identity()  argument
223 	lockdep_assert_held(&xe->irq.lock);  in gt_engine_identity()
238 		drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",  in gt_engine_identity()
240 		return 0;  in gt_engine_identity()
262 		WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",  in gt_other_irq_handler()
271 	struct xe_device *xe = tile_to_xe(tile);  in pick_engine_gt()  local
273 	if (MEDIA_VER(xe) < 13)  in pick_engine_gt()
299 	struct xe_device *xe = tile_to_xe(tile);  in gt_irq_handler()  local
306 	spin_lock(&xe->irq.lock);  in gt_irq_handler()
308 	for (bank = 0; bank < 2; bank++) {  in gt_irq_handler()
314 			identity[bit] = gt_engine_identity(xe, mmio, bank, bit);  in gt_irq_handler()
334 				if (xe->info.has_heci_gscfi && instance == OTHER_GSC_INSTANCE)  in gt_irq_handler()
335 					xe_heci_gsc_irq_handler(xe, intr_vec);  in gt_irq_handler()
342 	spin_unlock(&xe->irq.lock);  in gt_irq_handler()
351 	struct xe_device *xe = arg;  in xelp_irq_handler()  local
352 	struct xe_tile *tile = xe_device_get_root_tile(xe);  in xelp_irq_handler()
357 	if (!atomic_read(&xe->irq.enabled))  in xelp_irq_handler()
360 	master_ctl = xelp_intr_disable(xe);  in xelp_irq_handler()
362 		xelp_intr_enable(xe, false);  in xelp_irq_handler()
368 	xe_display_irq_handler(xe, master_ctl);  in xelp_irq_handler()
370 	gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);  in xelp_irq_handler()
372 	xelp_intr_enable(xe, false);  in xelp_irq_handler()
374 	xe_display_irq_enable(xe, gu_misc_iir);  in xelp_irq_handler()
379 static u32 dg1_intr_disable(struct xe_device *xe)  in dg1_intr_disable()  argument
381 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in dg1_intr_disable()
385 	xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0);  in dg1_intr_disable()
390 		return 0;  in dg1_intr_disable()
397 static void dg1_intr_enable(struct xe_device *xe, bool stall)  in dg1_intr_enable()  argument
399 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in dg1_intr_enable()
413 	struct xe_device *xe = arg;  in dg1_irq_handler()  local
415 	u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0;  in dg1_irq_handler()
422 	if (!atomic_read(&xe->irq.enabled))  in dg1_irq_handler()
425 	master_tile_ctl = dg1_intr_disable(xe);  in dg1_irq_handler()
427 		dg1_intr_enable(xe, false);  in dg1_irq_handler()
431 	for_each_tile(tile, xe, id) {  in dg1_irq_handler()
434 		if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0)  in dg1_irq_handler()
444 		if (master_ctl == REG_GENMASK(31, 0)) {  in dg1_irq_handler()
459 		if (id == 0) {  in dg1_irq_handler()
460 			if (xe->info.has_heci_cscfi)  in dg1_irq_handler()
461 				xe_heci_csc_irq_handler(xe, master_ctl);  in dg1_irq_handler()
462 			xe_display_irq_handler(xe, master_ctl);  in dg1_irq_handler()
463 			gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);  in dg1_irq_handler()
467 	dg1_intr_enable(xe, false);  in dg1_irq_handler()
468 	xe_display_irq_enable(xe, gu_misc_iir);  in dg1_irq_handler()
483 	xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0);  in gt_irq_reset()
484 	xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0);  in gt_irq_reset()
486 		xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0);  in gt_irq_reset()
489 	xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK,	~0);  in gt_irq_reset()
490 	xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK,	~0);  in gt_irq_reset()
492 		xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0);  in gt_irq_reset()
494 		xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0);  in gt_irq_reset()
496 		xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0);  in gt_irq_reset()
498 		xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0);  in gt_irq_reset()
499 	xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK,	~0);  in gt_irq_reset()
500 	xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK,	~0);  in gt_irq_reset()
501 	xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK,	~0);  in gt_irq_reset()
502 	if (ccs_mask & (BIT(0)|BIT(1)))  in gt_irq_reset()
503 		xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0);  in gt_irq_reset()
505 		xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0);  in gt_irq_reset()
510 		xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0);  in gt_irq_reset()
511 		xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0);  in gt_irq_reset()
512 		xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0);  in gt_irq_reset()
515 	xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0);  in gt_irq_reset()
516 	xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK,  ~0);  in gt_irq_reset()
517 	xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE,	 0);  in gt_irq_reset()
518 	xe_mmio_write32(mmio, GUC_SG_INTR_MASK,		~0);  in gt_irq_reset()
535 	if (tile->id == 0)  in dg1_irq_reset()
550 	xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0);  in dg1_irq_reset_mstr()
553 static void vf_irq_reset(struct xe_device *xe)  in vf_irq_reset()  argument
558 	xe_assert(xe, IS_SRIOV_VF(xe));  in vf_irq_reset()
560 	if (GRAPHICS_VERx100(xe) < 1210)  in vf_irq_reset()
561 		xelp_intr_disable(xe);  in vf_irq_reset()
563 		xe_assert(xe, xe_device_has_memirq(xe));  in vf_irq_reset()
565 	for_each_tile(tile, xe, id) {  in vf_irq_reset()
566 		if (xe_device_has_memirq(xe))  in vf_irq_reset()
573 static void xe_irq_reset(struct xe_device *xe)  in xe_irq_reset()  argument
578 	if (IS_SRIOV_VF(xe))  in xe_irq_reset()
579 		return vf_irq_reset(xe);  in xe_irq_reset()
581 	if (xe_device_uses_memirq(xe)) {  in xe_irq_reset()
582 		for_each_tile(tile, xe, id)  in xe_irq_reset()
586 	for_each_tile(tile, xe, id) {  in xe_irq_reset()
587 		if (GRAPHICS_VERx100(xe) >= 1210)  in xe_irq_reset()
593 	tile = xe_device_get_root_tile(xe);  in xe_irq_reset()
595 	xe_display_irq_reset(xe);  in xe_irq_reset()
602 	if (GRAPHICS_VERx100(xe) >= 1210) {  in xe_irq_reset()
603 		for_each_tile(tile, xe, id)  in xe_irq_reset()
608 static void vf_irq_postinstall(struct xe_device *xe)  in vf_irq_postinstall()  argument
613 	for_each_tile(tile, xe, id)  in vf_irq_postinstall()
614 		if (xe_device_has_memirq(xe))  in vf_irq_postinstall()
617 	if (GRAPHICS_VERx100(xe) < 1210)  in vf_irq_postinstall()
618 		xelp_intr_enable(xe, true);  in vf_irq_postinstall()
620 		xe_assert(xe, xe_device_has_memirq(xe));  in vf_irq_postinstall()
623 static void xe_irq_postinstall(struct xe_device *xe)  in xe_irq_postinstall()  argument
625 	if (IS_SRIOV_VF(xe))  in xe_irq_postinstall()
626 		return vf_irq_postinstall(xe);  in xe_irq_postinstall()
628 	if (xe_device_uses_memirq(xe)) {  in xe_irq_postinstall()
632 		for_each_tile(tile, xe, id)  in xe_irq_postinstall()
636 	xe_display_irq_postinstall(xe, xe_root_mmio_gt(xe));  in xe_irq_postinstall()
642 	unmask_and_enable(xe_device_get_root_tile(xe),  in xe_irq_postinstall()
646 	if (GRAPHICS_VERx100(xe) >= 1210)  in xe_irq_postinstall()
647 		dg1_intr_enable(xe, true);  in xe_irq_postinstall()
649 		xelp_intr_enable(xe, true);  in xe_irq_postinstall()
654 	struct xe_device *xe = arg;  in vf_mem_irq_handler()  local
658 	if (!atomic_read(&xe->irq.enabled))  in vf_mem_irq_handler()
661 	for_each_tile(tile, xe, id)  in vf_mem_irq_handler()
667 static irq_handler_t xe_irq_handler(struct xe_device *xe)  in xe_irq_handler()  argument
669 	if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe))  in xe_irq_handler()
672 	if (GRAPHICS_VERx100(xe) >= 1210)  in xe_irq_handler()
678 static int xe_irq_msi_request_irqs(struct xe_device *xe)  in xe_irq_msi_request_irqs()  argument
680 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_msi_request_irqs()
684 	irq_handler = xe_irq_handler(xe);  in xe_irq_msi_request_irqs()
686 		drm_err(&xe->drm, "No supported interrupt handler");  in xe_irq_msi_request_irqs()
690 	irq = pci_irq_vector(pdev, 0);  in xe_irq_msi_request_irqs()
691 	err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe);  in xe_irq_msi_request_irqs()
692 	if (err < 0) {  in xe_irq_msi_request_irqs()
693 		drm_err(&xe->drm, "Failed to request MSI IRQ %d\n", err);  in xe_irq_msi_request_irqs()
697 	return 0;  in xe_irq_msi_request_irqs()
700 static void xe_irq_msi_free(struct xe_device *xe)  in xe_irq_msi_free()  argument
702 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_msi_free()
705 	irq = pci_irq_vector(pdev, 0);  in xe_irq_msi_free()
706 	free_irq(irq, xe);  in xe_irq_msi_free()
711 	struct xe_device *xe = arg;  in irq_uninstall()  local
713 	if (!atomic_xchg(&xe->irq.enabled, 0))  in irq_uninstall()
716 	xe_irq_reset(xe);  in irq_uninstall()
718 	if (xe_device_has_msix(xe))  in irq_uninstall()
719 		xe_irq_msix_free(xe);  in irq_uninstall()
721 		xe_irq_msi_free(xe);  in irq_uninstall()
724 int xe_irq_init(struct xe_device *xe)  in xe_irq_init()  argument
726 	spin_lock_init(&xe->irq.lock);  in xe_irq_init()
728 	return xe_irq_msix_init(xe);  in xe_irq_init()
731 int xe_irq_install(struct xe_device *xe)  in xe_irq_install()  argument
733 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_install()
738 	xe_irq_reset(xe);  in xe_irq_install()
740 	if (xe_device_has_msix(xe)) {  in xe_irq_install()
741 		nvec = xe->irq.msix.nvec;  in xe_irq_install()
746 	if (err < 0) {  in xe_irq_install()
747 		drm_err(&xe->drm, "Failed to allocate IRQ vectors: %d\n", err);  in xe_irq_install()
751 	err = xe_device_has_msix(xe) ? xe_irq_msix_request_irqs(xe) :  in xe_irq_install()
752 					xe_irq_msi_request_irqs(xe);  in xe_irq_install()
756 	atomic_set(&xe->irq.enabled, 1);  in xe_irq_install()
758 	xe_irq_postinstall(xe);  in xe_irq_install()
760 	return devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe);  in xe_irq_install()
763 static void xe_irq_msi_synchronize_irq(struct xe_device *xe)  in xe_irq_msi_synchronize_irq()  argument
765 	synchronize_irq(to_pci_dev(xe->drm.dev)->irq);  in xe_irq_msi_synchronize_irq()
768 void xe_irq_suspend(struct xe_device *xe)  in xe_irq_suspend()  argument
770 	atomic_set(&xe->irq.enabled, 0); /* no new irqs */  in xe_irq_suspend()
773 	if (xe_device_has_msix(xe))  in xe_irq_suspend()
774 		xe_irq_msix_synchronize_irq(xe);  in xe_irq_suspend()
776 		xe_irq_msi_synchronize_irq(xe);  in xe_irq_suspend()
777 	xe_irq_reset(xe); /* turn irqs off */  in xe_irq_suspend()
780 void xe_irq_resume(struct xe_device *xe)  in xe_irq_resume()  argument
790 	atomic_set(&xe->irq.enabled, 1);  in xe_irq_resume()
791 	xe_irq_reset(xe);  in xe_irq_resume()
792 	xe_irq_postinstall(xe); /* turn irqs on */  in xe_irq_resume()
794 	for_each_gt(gt, xe, id)  in xe_irq_resume()
801 	GUC2HOST_MSIX = 0,
807 static int xe_irq_msix_init(struct xe_device *xe)  in xe_irq_msix_init()  argument
809 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_msix_init()
813 		return 0;  /* MSI */  in xe_irq_msix_init()
815 	if (nvec < 0) {  in xe_irq_msix_init()
816 		drm_err(&xe->drm, "Failed getting MSI-X vectors count: %d\n", nvec);  in xe_irq_msix_init()
820 	xe->irq.msix.nvec = nvec;  in xe_irq_msix_init()
821 	xa_init_flags(&xe->irq.msix.indexes, XA_FLAGS_ALLOC);  in xe_irq_msix_init()
822 	return 0;  in xe_irq_msix_init()
827 	struct xe_device *xe = arg;  in guc2host_irq_handler()  local
831 	if (!atomic_read(&xe->irq.enabled))  in guc2host_irq_handler()
834 	for_each_tile(tile, xe, id)  in guc2host_irq_handler()
844 	struct xe_device *xe = arg;  in xe_irq_msix_default_hwe_handler()  local
851 	if (!atomic_read(&xe->irq.enabled))  in xe_irq_msix_default_hwe_handler()
854 	for_each_tile(tile, xe, tile_id) {  in xe_irq_msix_default_hwe_handler()
859 		for_each_gt(gt, xe, gt_id) {  in xe_irq_msix_default_hwe_handler()
871 static int xe_irq_msix_alloc_vector(struct xe_device *xe, void *irq_buf,  in xe_irq_msix_alloc_vector()  argument
878 	limit = (dynamic_msix) ? XA_LIMIT(NUM_OF_STATIC_MSIX, xe->irq.msix.nvec - 1) :  in xe_irq_msix_alloc_vector()
880 	ret = xa_alloc(&xe->irq.msix.indexes, &id, irq_buf, limit, GFP_KERNEL);  in xe_irq_msix_alloc_vector()
887 	return 0;  in xe_irq_msix_alloc_vector()
890 static void xe_irq_msix_release_vector(struct xe_device *xe, u16 msix)  in xe_irq_msix_release_vector()  argument
892 	xa_erase(&xe->irq.msix.indexes, msix);  in xe_irq_msix_release_vector()
895 static int xe_irq_msix_request_irq_internal(struct xe_device *xe, irq_handler_t handler,  in xe_irq_msix_request_irq_internal()  argument
898 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_msix_request_irq_internal()
902 	if (irq < 0)  in xe_irq_msix_request_irq_internal()
906 	if (ret < 0)  in xe_irq_msix_request_irq_internal()
909 	return 0;  in xe_irq_msix_request_irq_internal()
912 int xe_irq_msix_request_irq(struct xe_device *xe, irq_handler_t handler, void *irq_buf,  in xe_irq_msix_request_irq()  argument
917 	ret = xe_irq_msix_alloc_vector(xe, irq_buf, dynamic_msix, msix);  in xe_irq_msix_request_irq()
921 	ret = xe_irq_msix_request_irq_internal(xe, handler, irq_buf, name, *msix);  in xe_irq_msix_request_irq()
923 		drm_err(&xe->drm, "Failed to request IRQ for MSI-X %u\n", *msix);  in xe_irq_msix_request_irq()
924 		xe_irq_msix_release_vector(xe, *msix);  in xe_irq_msix_request_irq()
928 	return 0;  in xe_irq_msix_request_irq()
931 void xe_irq_msix_free_irq(struct xe_device *xe, u16 msix)  in xe_irq_msix_free_irq()  argument
933 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_msix_free_irq()
937 	irq_buf = xa_load(&xe->irq.msix.indexes, msix);  in xe_irq_msix_free_irq()
942 	if (irq < 0) {  in xe_irq_msix_free_irq()
943 		drm_err(&xe->drm, "MSI-X %u can't be released, there is no matching IRQ\n", msix);  in xe_irq_msix_free_irq()
948 	xe_irq_msix_release_vector(xe, msix);  in xe_irq_msix_free_irq()
951 int xe_irq_msix_request_irqs(struct xe_device *xe)  in xe_irq_msix_request_irqs()  argument
957 	err = xe_irq_msix_request_irq(xe, guc2host_irq_handler, xe,  in xe_irq_msix_request_irqs()
963 	err = xe_irq_msix_request_irq(xe, xe_irq_msix_default_hwe_handler, xe,  in xe_irq_msix_request_irqs()
966 		xe_irq_msix_free_irq(xe, GUC2HOST_MSIX);  in xe_irq_msix_request_irqs()
970 	return 0;  in xe_irq_msix_request_irqs()
973 void xe_irq_msix_free(struct xe_device *xe)  in xe_irq_msix_free()  argument
978 	xa_for_each(&xe->irq.msix.indexes, msix, dummy)  in xe_irq_msix_free()
979 		xe_irq_msix_free_irq(xe, msix);  in xe_irq_msix_free()
980 	xa_destroy(&xe->irq.msix.indexes);  in xe_irq_msix_free()
983 void xe_irq_msix_synchronize_irq(struct xe_device *xe)  in xe_irq_msix_synchronize_irq()  argument
985 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_msix_synchronize_irq()
989 	xa_for_each(&xe->irq.msix.indexes, msix, dummy)  in xe_irq_msix_synchronize_irq()