1 // SPDX-License-Identifier: GPL-2.0
3  * Implement the AER root port service driver. The driver registers an IRQ
4  * handler. When a root port triggers an AER interrupt, the IRQ handler
5  * collects root port status and schedules work.
11  * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
21 #include <linux/pci-acpi.h>
49 	struct pci_dev *rpd;		/* Root Port device */
60 	 * at its link partner (e.g. root port) because the errors will be
79 	 * Fields for Root ports & root complex event collectors only, these
81 	 * messages received by the root port / event collector, INCLUDING the
115 	pcie_aer_disable = 1;  in pci_no_aer()
126 #define ECRC_POLICY_OFF     1		/* ECRC off for performance */
138  * enable_ecrc_checking - enable PCIe ECRC checking for a device
145 	int aer = dev->aer_cap;  in enable_ecrc_checking()
149 		return -ENODEV;  in enable_ecrc_checking()
162  * disable_ecrc_checking - disables PCIe ECRC checking for a device
169 	int aer = dev->aer_cap;  in disable_ecrc_checking()
173 		return -ENODEV;  in disable_ecrc_checking()
183  * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based
207  * pcie_ecrc_get_policy - parse kernel command-line ecrc option
227 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in pcie_aer_is_native()
229 	if (!dev->aer_cap)  in pcie_aer_is_native()
232 	return pcie_ports_native || host->native_aer;  in pcie_aer_is_native()
241 		return -EIO;  in pci_enable_pcie_error_reporting()
249 	int aer = dev->aer_cap;  in pci_aer_clear_nonfatal_status()
253 		return -EIO;  in pci_aer_clear_nonfatal_status()
268 	int aer = dev->aer_cap;  in pci_aer_clear_fatal_status()
283  * pci_aer_raw_clear_status - Clear AER error registers.
293 	int aer = dev->aer_cap;  in pci_aer_raw_clear_status()
298 		return -EIO;  in pci_aer_raw_clear_status()
319 		return -EIO;  in pci_aer_clear_status()
326 	int aer = dev->aer_cap;  in pci_save_aer_state()
337 	cap = &save_state->cap.data[0];  in pci_save_aer_state()
348 	int aer = dev->aer_cap;  in pci_restore_aer_state()
359 	cap = &save_state->cap.data[0];  in pci_restore_aer_state()
372 	dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);  in pci_aer_init()
373 	if (!dev->aer_cap)  in pci_aer_init()
376 	dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL);  in pci_aer_init()
380 	 * PCI_ERR_COR_MASK, and PCI_ERR_CAP.  Root and Root Complex Event  in pci_aer_init()
381 	 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r5.0, sec  in pci_aer_init()
397 	kfree(dev->aer_stats);  in pci_aer_exit()
398 	dev->aer_stats = NULL;  in pci_aer_exit()
402 #define AER_AGENT_REQUESTER		1
420 #define AER_DATA_LINK_LAYER_ERROR	1
440 	"Uncorrectable (Non-Fatal)",
536 	u64 *stats = pdev->aer_stats->stats_array;			\
539 	for (i = 0; i < ARRAY_SIZE(pdev->aer_stats->stats_array); i++) {\
550 			     pdev->aer_stats->total_field);		\
571 	return sysfs_emit(buf, "%llu\n", pdev->aer_stats->field);	\
598 	if (!pdev->aer_stats)  in aer_stats_attrs_are_visible()
608 	return a->mode;  in aer_stats_attrs_are_visible()
619 	unsigned long status = info->status & ~info->mask;  in pci_dev_aer_stats_incr()
620 	int i, max = -1;  in pci_dev_aer_stats_incr()
622 	struct aer_stats *aer_stats = pdev->aer_stats;  in pci_dev_aer_stats_incr()
627 	switch (info->severity) {  in pci_dev_aer_stats_incr()
629 		aer_stats->dev_total_cor_errs++;  in pci_dev_aer_stats_incr()
630 		counter = &aer_stats->dev_cor_errs[0];  in pci_dev_aer_stats_incr()
634 		aer_stats->dev_total_nonfatal_errs++;  in pci_dev_aer_stats_incr()
635 		counter = &aer_stats->dev_nonfatal_errs[0];  in pci_dev_aer_stats_incr()
639 		aer_stats->dev_total_fatal_errs++;  in pci_dev_aer_stats_incr()
640 		counter = &aer_stats->dev_fatal_errs[0];  in pci_dev_aer_stats_incr()
652 	struct aer_stats *aer_stats = pdev->aer_stats;  in pci_rootport_aer_stats_incr()
657 	if (e_src->status & PCI_ERR_ROOT_COR_RCV)  in pci_rootport_aer_stats_incr()
658 		aer_stats->rootport_total_cor_errs++;  in pci_rootport_aer_stats_incr()
660 	if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {  in pci_rootport_aer_stats_incr()
661 		if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)  in pci_rootport_aer_stats_incr()
662 			aer_stats->rootport_total_fatal_errs++;  in pci_rootport_aer_stats_incr()
664 			aer_stats->rootport_total_nonfatal_errs++;  in pci_rootport_aer_stats_incr()
672 	unsigned long status = info->status & ~info->mask;  in __aer_print_error()
676 	if (info->severity == AER_CORRECTABLE) {  in __aer_print_error()
689 		pci_printk(level, dev, "   [%2d] %-22s%s\n", i, errmsg,  in __aer_print_error()
690 				info->first_error == i ? " (First)" : "");  in __aer_print_error()
701 	if (!info->status) {  in aer_print_error()
702 		pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",  in aer_print_error()
703 			aer_error_severity_string[info->severity]);  in aer_print_error()
707 	layer = AER_GET_LAYER_ERROR(info->severity, info->status);  in aer_print_error()
708 	agent = AER_GET_AGENT(info->severity, info->status);  in aer_print_error()
710 	level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR;  in aer_print_error()
712 	pci_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",  in aer_print_error()
713 		   aer_error_severity_string[info->severity],  in aer_print_error()
717 		   dev->vendor, dev->device, info->status, info->mask);  in aer_print_error()
721 	if (info->tlp_header_valid)  in aer_print_error()
722 		pcie_print_tlp_log(dev, &info->tlp, dev_fmt("  "));  in aer_print_error()
725 	if (info->id && info->error_dev_num > 1 && info->id == id)  in aer_print_error()
728 	trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask),  in aer_print_error()
729 			info->severity, info->tlp_header_valid, &info->tlp);  in aer_print_error()
734 	u8 bus = info->id >> 8;  in aer_print_port_info()
735 	u8 devfn = info->id & 0xff;  in aer_print_port_info()
738 		 info->multi_error_valid ? "Multiple " : "",  in aer_print_port_info()
739 		 aer_error_severity_string[info->severity],  in aer_print_port_info()
740 		 pci_domain_nr(dev->bus), bus, PCI_SLOT(devfn),  in aer_print_port_info()
767 		status = aer->cor_status;  in pci_print_aer()
768 		mask = aer->cor_mask;  in pci_print_aer()
770 		status = aer->uncor_status;  in pci_print_aer()
771 		mask = aer->uncor_mask;  in pci_print_aer()
782 	info.first_error = PCI_ERR_CAP_FEP(aer->cap_control);  in pci_print_aer()
791 			aer->uncor_severity);  in pci_print_aer()
794 		pcie_print_tlp_log(dev, &aer->header_log, dev_fmt("  "));  in pci_print_aer()
796 	trace_aer_event(dev_name(&dev->dev), (status & ~mask),  in pci_print_aer()
797 			aer_severity, tlp_header_valid, &aer->header_log);  in pci_print_aer()
802  * add_error_device - list device to be handled
808 	if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {  in add_error_device()
809 		e_info->dev[e_info->error_dev_num] = pci_dev_get(dev);  in add_error_device()
810 		e_info->error_dev_num++;  in add_error_device()
813 	return -ENOSPC;  in add_error_device()
817  * is_error_source - check whether the device is source of reported error
823 	int aer = dev->aer_cap;  in is_error_source()
829 	 * reported by root port.  in is_error_source()
831 	if ((PCI_BUS_NUM(e_info->id) != 0) &&  in is_error_source()
832 	    !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {  in is_error_source()
834 		if (e_info->id == pci_dev_id(dev))  in is_error_source()
838 		if (!e_info->multi_error_valid)  in is_error_source()
844 	 *      1) bus id is equal to 0. Some ports might lose the bus  in is_error_source()
850 	if (atomic_read(&dev->enable_cnt) == 0)  in is_error_source()
862 	if (e_info->severity == AER_CORRECTABLE) {  in is_error_source()
884 			return 1;  in find_device_iter()
888 		if (!e_info->multi_error_valid)  in find_device_iter()
889 			return 1;  in find_device_iter()
895  * find_source_device - search through device hierarchy for source device
896  * @parent: pointer to Root Port pci_dev data structure
901  * Invoked by DPC when error is detected at the Root Port.
904  * e_info->error_dev_num and e_info->dev[], based on the given information.
913 	e_info->error_dev_num = 0;  in find_source_device()
915 	/* Is Root Port an agent that sends error message? */  in find_source_device()
923 		pci_walk_bus(parent->subordinate, find_device_iter, e_info);  in find_source_device()
925 	if (!e_info->error_dev_num) {  in find_source_device()
926 		u8 bus = e_info->id >> 8;  in find_source_device()
927 		u8 devfn = e_info->id & 0xff;  in find_source_device()
930 			 pci_domain_nr(parent->bus), bus, PCI_SLOT(devfn),  in find_source_device()
940  * pci_aer_unmask_internal_errors - unmask internal errors
951 	int aer = dev->aer_cap;  in pci_aer_unmask_internal_errors()
970 	if (dev->devfn != PCI_DEVFN(0, 0))  in is_cxl_mem_dev()
977 	if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)  in is_cxl_mem_dev()
985 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in cxl_error_is_native()
987 	return (pcie_ports_native || host->native_aer);  in cxl_error_is_native()
992 	if (info->severity == AER_CORRECTABLE)  in is_internal_error()
993 		return info->status & PCI_ERR_COR_INTERNAL;  in is_internal_error()
995 	return info->status & PCI_ERR_UNC_INTN;  in is_internal_error()
1006 	/* protect dev->driver */  in cxl_rch_handle_error_iter()
1007 	device_lock(&dev->dev);  in cxl_rch_handle_error_iter()
1009 	err_handler = dev->driver ? dev->driver->err_handler : NULL;  in cxl_rch_handle_error_iter()
1013 	if (info->severity == AER_CORRECTABLE) {  in cxl_rch_handle_error_iter()
1014 		if (err_handler->cor_error_detected)  in cxl_rch_handle_error_iter()
1015 			err_handler->cor_error_detected(dev);  in cxl_rch_handle_error_iter()
1016 	} else if (err_handler->error_detected) {  in cxl_rch_handle_error_iter()
1017 		if (info->severity == AER_NONFATAL)  in cxl_rch_handle_error_iter()
1018 			err_handler->error_detected(dev, pci_channel_io_normal);  in cxl_rch_handle_error_iter()
1019 		else if (info->severity == AER_FATAL)  in cxl_rch_handle_error_iter()
1020 			err_handler->error_detected(dev, pci_channel_io_frozen);  in cxl_rch_handle_error_iter()
1023 	device_unlock(&dev->dev);  in cxl_rch_handle_error_iter()
1031 	 * RCH's downstream port. Check and handle them in the CXL.mem  in cxl_rch_handle_error()
1046 	/* Non-zero terminates iteration */  in handles_cxl_error_iter()
1077  * pci_aer_handle_error - handle logging error into an event log
1081  * Invoked when an error being detected by Root Port.
1085 	int aer = dev->aer_cap;  in pci_aer_handle_error()
1087 	if (info->severity == AER_CORRECTABLE) {  in pci_aer_handle_error()
1094 					info->status);  in pci_aer_handle_error()
1096 			struct pci_driver *pdrv = dev->driver;  in pci_aer_handle_error()
1098 			if (pdrv && pdrv->err_handler &&  in pci_aer_handle_error()
1099 			    pdrv->err_handler->cor_error_detected)  in pci_aer_handle_error()
1100 				pdrv->err_handler->cor_error_detected(dev);  in pci_aer_handle_error()
1103 	} else if (info->severity == AER_NONFATAL)  in pci_aer_handle_error()
1105 	else if (info->severity == AER_FATAL)  in pci_aer_handle_error()
1186 	if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1,  in aer_recover_queue()
1197  * aer_get_device_error_info - read error status from dev and store it to info
1201  * Return 1 on success, 0 on error.
1208 	int aer = dev->aer_cap;  in aer_get_device_error_info()
1212 	info->status = 0;  in aer_get_device_error_info()
1213 	info->tlp_header_valid = 0;  in aer_get_device_error_info()
1219 	if (info->severity == AER_CORRECTABLE) {  in aer_get_device_error_info()
1221 			&info->status);  in aer_get_device_error_info()
1223 			&info->mask);  in aer_get_device_error_info()
1224 		if (!(info->status & ~info->mask))  in aer_get_device_error_info()
1229 		   info->severity == AER_NONFATAL) {  in aer_get_device_error_info()
1233 			&info->status);  in aer_get_device_error_info()
1235 			&info->mask);  in aer_get_device_error_info()
1236 		if (!(info->status & ~info->mask))  in aer_get_device_error_info()
1241 		info->first_error = PCI_ERR_CAP_FEP(aercc);  in aer_get_device_error_info()
1243 		if (info->status & AER_LOG_TLP_MASKS) {  in aer_get_device_error_info()
1244 			info->tlp_header_valid = 1;  in aer_get_device_error_info()
1248 					  &info->tlp);  in aer_get_device_error_info()
1252 	return 1;  in aer_get_device_error_info()
1260 	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {  in aer_process_err_devices()
1261 		if (aer_get_device_error_info(e_info->dev[i], e_info))  in aer_process_err_devices()
1262 			aer_print_error(e_info->dev[i], e_info);  in aer_process_err_devices()
1264 	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {  in aer_process_err_devices()
1265 		if (aer_get_device_error_info(e_info->dev[i], e_info))  in aer_process_err_devices()
1266 			handle_error_source(e_info->dev[i], e_info);  in aer_process_err_devices()
1271  * aer_isr_one_error - consume an error detected by root port
1272  * @rpc: pointer to the root port which holds an error
1278 	struct pci_dev *pdev = rpc->rpd;  in aer_isr_one_error()
1287 	if (e_src->status & PCI_ERR_ROOT_COR_RCV) {  in aer_isr_one_error()
1288 		e_info.id = ERR_COR_ID(e_src->id);  in aer_isr_one_error()
1291 		if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV)  in aer_isr_one_error()
1292 			e_info.multi_error_valid = 1;  in aer_isr_one_error()
1301 	if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {  in aer_isr_one_error()
1302 		e_info.id = ERR_UNCOR_ID(e_src->id);  in aer_isr_one_error()
1304 		if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)  in aer_isr_one_error()
1309 		if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV)  in aer_isr_one_error()
1310 			e_info.multi_error_valid = 1;  in aer_isr_one_error()
1322  * aer_isr - consume errors detected by root port
1323  * @irq: IRQ assigned to Root Port
1324  * @context: pointer to Root Port data structure
1326  * Invoked, as DPC, when root port records new detected error
1334 	if (kfifo_is_empty(&rpc->aer_fifo))  in aer_isr()
1337 	while (kfifo_get(&rpc->aer_fifo, &e_src))  in aer_isr()
1343  * aer_irq - Root Port's ISR
1344  * @irq: IRQ assigned to Root Port
1345  * @context: pointer to Root Port data structure
1347  * Invoked when Root Port detects AER messages.
1353 	struct pci_dev *rp = rpc->rpd;  in aer_irq()
1354 	int aer = rp->aer_cap;  in aer_irq()
1364 	if (!kfifo_put(&rpc->aer_fifo, e_src))  in aer_irq()
1372 	int aer = pdev->aer_cap;  in aer_enable_irq()
1375 	/* Enable Root Port's interrupt in response to error messages */  in aer_enable_irq()
1383 	int aer = pdev->aer_cap;  in aer_disable_irq()
1386 	/* Disable Root's interrupt in response to error messages */  in aer_disable_irq()
1393  * aer_enable_rootport - enable Root Port's interrupts when receiving messages
1394  * @rpc: pointer to a Root Port data structure
1396  * Invoked when PCIe bus loads AER service driver.
1400 	struct pci_dev *pdev = rpc->rpd;  in aer_enable_rootport()
1401 	int aer = pdev->aer_cap;  in aer_enable_rootport()
1405 	/* Clear PCIe Capability's Device Status */  in aer_enable_rootport()
1425  * aer_disable_rootport - disable Root Port's interrupts when receiving messages
1426  * @rpc: pointer to a Root Port data structure
1428  * Invoked when PCIe bus unloads AER service driver.
1432 	struct pci_dev *pdev = rpc->rpd;  in aer_disable_rootport()
1433 	int aer = pdev->aer_cap;  in aer_disable_rootport()
1438 	/* Clear Root's error status reg */  in aer_disable_rootport()
1444  * aer_remove - clean up resources
1457  * aer_probe - initialize resources
1466 	struct device *device = &dev->device;  in aer_probe()
1467 	struct pci_dev *port = dev->port;  in aer_probe()  local
1474 	/* Limit to Root Ports or Root Complex Event Collectors */  in aer_probe()
1475 	if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&  in aer_probe()
1476 	    (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))  in aer_probe()
1477 		return -ENODEV;  in aer_probe()
1481 		return -ENOMEM;  in aer_probe()
1483 	rpc->rpd = port;  in aer_probe()
1484 	INIT_KFIFO(rpc->aer_fifo);  in aer_probe()
1487 	status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,  in aer_probe()
1490 		pci_err(port, "request AER IRQ %d failed\n", dev->irq);  in aer_probe()
1494 	cxl_rch_enable_rcec(port);  in aer_probe()
1496 	pci_info(port, "enabled with IRQ %d\n", dev->irq);  in aer_probe()
1517  * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP
1518  * @dev: pointer to Root Port, RCEC, or RCiEP
1520  * Invoked by Port Bus driver when performing reset.
1525 	struct pci_dev *root;  in aer_root_reset()  local
1527 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in aer_root_reset()
1532 	 * Only Root Ports and RCECs have AER Root Command and Root Status  in aer_root_reset()
1537 		root = dev->rcec;  in aer_root_reset()
1539 		root = pcie_find_root_port(dev);  in aer_root_reset()
1543 	 * an RCEC visible to us, so dev->rcec ("root") may be NULL.  In  in aer_root_reset()
1546 	aer = root ? root->aer_cap : 0;  in aer_root_reset()
1548 	if ((host->native_aer || pcie_ports_native) && aer)  in aer_root_reset()
1549 		aer_disable_irq(root);  in aer_root_reset()
1559 		pci_info(dev, "%s Port link has been reset (%d)\n",  in aer_root_reset()
1560 			pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);  in aer_root_reset()
1563 	if ((host->native_aer || pcie_ports_native) && aer) {  in aer_root_reset()
1564 		/* Clear Root Error Status */  in aer_root_reset()
1565 		pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, &reg32);  in aer_root_reset()
1566 		pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32);  in aer_root_reset()
1568 		aer_enable_irq(root);  in aer_root_reset()
1586  * pcie_aer_init - register AER root service driver
1588  * Invoked when AER root service driver is loaded.
1593 		return -ENXIO;  in pcie_aer_init()