1 /* SPDX-License-Identifier: GPL-2.0 */
19 #include <linux/io-64-nonatomic-lo-hi.h>
20 #include <linux/io-64-nonatomic-hi-lo.h>
22 /* Code sharing between pci-quirks and xhci hcd */
23 #include	"xhci-ext-caps.h"
24 #include "pci-quirks.h"
26 #include "xhci-port.h"
27 #include "xhci-caps.h"
35 /* Max number of USB devices for any host controller - limit in section 6.1 */
37 /* Section 5.3.3 - MaxPorts */
47  * struct xhci_cap_regs - xHCI Host Controller Capability Registers.
49  * @hcs_params1:	HCSPARAMS1 - Structural Parameters 1
50  * @hcs_params2:	HCSPARAMS2 - Structural Parameters 2
51  * @hcs_params3:	HCSPARAMS3 - Structural Parameters 3
52  * @hcc_params:		HCCPARAMS - Capability Parameters
53  * @db_off:		DBOFF - Doorbell array offset
54  * @run_regs_off:	RTSOFF - Runtime register space offset
66 	/* Reserved up to (CAPLENGTH - 0x1C) */
73 #define PORTPMSC	1
78  * struct xhci_op_regs - xHCI Host Controller Operational Registers.
79  * @command:		USBCMD - xHC command register
80  * @status:		USBSTS - xHC status register
85  * @cmd_ring:		CRP - 64-bit Command Ring Pointer
86  * @dcbaa_ptr:		DCBAAP - 64-bit Device Context Base Address Array Pointer
87  * @config_reg:		CONFIG - Configure Register
88  * @port_status_base:	PORTSCn - base address for Port Status and Control
93  * @port_power_base:	PORTPMSCn - base address for
95  * @port_link_base:	PORTLIn - base address for Port Link Info (current
107 	/* rsvd: offset 0x20-2F */
111 	/* rsvd: offset 0x3C-3FF */
113 	/* port 1 registers, which serve as a base address for other ports */
118 	/* registers for ports 2-255 */
122 /* USBCMD - USB command - command bitmasks */
123 /* start/stop HC execution - do not write unless HC is halted*/
125 /* Reset HC - resets internal HC state machine and all registers (except
129 #define CMD_RESET	(1 << 1)
130 /* Event Interrupt Enable - a '1' allows interrupts from the host controller */
132 /* Host System Error Interrupt Enable - get out-of-band signal for HC errors */
135 /* light reset (port status stays unchanged) - reset completed when this is 0 */
136 #define CMD_LRESET	(1 << 7)
138 #define CMD_CSS		(1 << 8)
139 #define CMD_CRS		(1 << 9)
140 /* Enable Wrap Event - '1' means xHC generates an event when MFINDEX wraps. */
142 /* MFINDEX power management - '1' means xHC can stop MFINDEX counter if all root
143  * hubs are in U3 (selective suspend), disconnect, disabled, or powered-off.
145  * disabled, or powered-off state.
147 #define CMD_PM_INDEX	(1 << 11)
149 #define CMD_ETE		(1 << 14)
155 /* IMAN - Interrupt Management Register */
156 #define IMAN_IE		(1 << 1)
157 #define IMAN_IP		(1 << 0)
159 /* USBSTS - USB status - status bitmasks */
160 /* HC not running - set to 1 when run/stop bit is cleared. */
163 #define STS_FATAL	(1 << 2)
164 /* event interrupt - clear this prior to clearing any IP flags in IR set*/
165 #define STS_EINT	(1 << 3)
167 #define STS_PORT	(1 << 4)
169 /* save state status - '1' means xHC is saving state */
170 #define STS_SAVE	(1 << 8)
171 /* restore state status - '1' means xHC is restoring state */
172 #define STS_RESTORE	(1 << 9)
174 #define STS_SRE		(1 << 10)
177 /* true: internal Host Controller Error - SW needs to reset and reinitialize */
178 #define STS_HCE		(1 << 12)
182  * DNCTRL - Device Notification Control Register - dev_notification bitmasks
187 #define ENABLE_DEV_NOTE(x)	(1 << (x))
191 #define	DEV_NOTE_FWAKE		ENABLE_DEV_NOTE(1)
193 /* CRCR - Command Ring Control Register - cmd_ring bitmasks */
196 #define CMD_RING_PAUSE		(1 << 1)
197 /* stop ring immediately - abort the currently executing command */
198 #define CMD_RING_ABORT		(1 << 2)
200 #define CMD_RING_RUNNING	(1 << 3)
202 /* Command Ring pointer - bit mask for the lower 32 bits. */
205 /* CONFIG - Configure Register - config_reg bitmasks */
206 /* bits 0:7 - maximum number of device slots enabled (NumSlotsEn) */
209 #define CONFIG_U3E		(1 << 8)
211 #define CONFIG_CIE		(1 << 9)
212 /* bits 10:31 - reserved and should be preserved */
215  * struct xhci_intr_reg - Interrupt Register Set
216  * @irq_pending:	IMAN - Interrupt Management Register.  Used to enable
218  * @irq_control:	IMOD - Interrupt Moderation Register.
224  * Each interrupter (defined by a MSI-X vector) has an event ring and an Event
243 /* THIS IS BUGGY - FIXME - IP IS WRITE 1 TO CLEAR */
251  * Default is 4000 (1 ms).
254 /* Counter used to count down the time to the next interrupt - HW use only */
265 /* Dequeue ERST Segment Index (DESI) - Segment number (or alias)
269 /* Event Handler Busy (EHB) - is the event ring scheduled to be serviced by
272 #define ERST_EHB		(1 << 3)
278  * 		MFINDEX - current microframe number
293  * Bits  0 -  7: Endpoint target
294  * Bits  8 - 15: RsvdZ
295  * Bits 16 - 31: Stream ID
303 #define DB_VALUE(ep, stream)	((((ep) + 1) & 0xff) | ((stream) << 16))
339  * Slot Context - section 6.2.1.1.  This assumes the HC uses 32-byte context
340  * structures.  If the HC uses 64-byte contexts, there is an additional 32 bytes
353 /* Route String - 0:19 */
355 /* Device speed - values defined by PORTSC Device Speed field - 20:23 */
359 /* Is this LS/FS device connected through a HS hub? - bit 25 */
361 /* Set if the device is a hub - bit 26 */
363 /* Index of the last valid endpoint context in this device context - 27:31 */
366 #define LAST_CTX_TO_EP_NUM(p)	(((p) >> 27) - 1)
367 #define SLOT_FLAG	(1 << 0)
368 #define EP0_FLAG	(1 << 1)
371 /* Max Exit Latency (ms) - worst case time to wake up all links in dev path */
382  * TT Hub Slot ID - for low or full speed devices attached to a high-speed hub
384  * this low or full-speed device.  '0' if attached to root hub port.
388  * The number of the downstream facing port of the high-speed hub
396 /* USB device address - assigned by the HC */
405 #define SLOT_STATE_DEFAULT	1
415  * @deq:	64-bit ring dequeue pointer address.  If the endpoint only
423  * Endpoint Context - section 6.2.1.2.  This assumes the HC uses 32-byte context
424  * structures.  If the HC uses 64-byte contexts, there is an additional 32 bytes
432 	/* offset 0x14 - 0x1f reserved for HC internal use */
438  * Endpoint State - bits 0:2
439  * 0 - disabled
440  * 1 - running
441  * 2 - halted due to halt condition - ok to manipulate endpoint ring
442  * 3 - stopped
443  * 4 - TRB error
444  * 5-7 - reserved
448 #define EP_STATE_RUNNING	1
452 #define GET_EP_CTX_STATE(ctx)	(le32_to_cpu((ctx)->ep_info) & EP_STATE_MASK)
454 /* Mult - Max number of burtst within an interval, in EP companion desc. */
459 /* Interval - period between requests to an endpoint - 125u increments. */
461 #define EP_INTERVAL_TO_UFRAMES(p)	(1 << (((p) >> 16) & 0xff))
467 #define	EP_HAS_LSA		(1 << 15)
468 /* hosts with LEC=1 use bits 31:24 as ESIT high bits. */
473  * Force Event - generate transfer events for all TRBs for this endpoint
477 #define ERROR_COUNT(p)	(((p) & 0x3) << 1)
480 #define ISOC_OUT_EP	1
488 /* bit 7 is Host Initiate Disable - for disabling stream selection */
502 #define EP_CTX_CYCLE_MASK		(1 << 0)
520 	(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))
522 	(le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1)))
525  * It's useful to pre-allocate these for commands that cannot fail due to
526  * out-of-memory errors, like freeing streams.
550 	/* 64-bit stream ring address, cycle state, and stream type */
552 	/* offset 0x14 - 0x1f reserved for HC internal use */
556 /* Stream Context Types (section 6.4.1) - bits 3:1 of stream ctx deq ptr */
557 #define	SCT_FOR_CTX(p)		(((p) & 0x7) << 1)
558 #define	CTX_TO_SCT(p)		(((p) >> 1) & 0x7)
562 #define	SCT_PRI_TR		1
593  * multi-TT hub) as a separate bandwidth domain.  The direct memory interface
597 	/* ep_interval is zero-based */
599 	/* mult and num_packets are one-based */
612 #define	FS_BLOCK	1
646 /* Percentage of bus bandwidth reserved for non-periodic transfers */
663 #define SET_DEQ_PENDING		(1 << 0)
664 #define EP_HALTED		(1 << 1)	/* For stall handling */
665 #define EP_STOP_CMD_PENDING	(1 << 2)	/* For URB cancellation */
667 #define EP_GETTING_STREAMS	(1 << 3)
668 #define EP_HAS_STREAMS		(1 << 4)
670 #define EP_GETTING_NO_STREAMS	(1 << 5)
671 #define EP_HARD_CLEAR_TOGGLE	(1 << 6)
672 #define EP_SOFT_CLEAR_TOGGLE	(1 << 7)
674 #define EP_CLEARING_TT		(1 << 8)
675 	/* ----  Related to URB cancellation ---- */
789  * @dev_context_ptr	array of 64-bit DMA addresses for device contexts
792 	/* 64-bit device addresses; we only write 32-bit addresses */
797 /* TODO: write function to set the 64-bit device DMA address */
805 	/* 64-bit buffer address, or immediate data */
816 #define TRB_TO_EP_ID(p)		(((p) >> 16) & 0x1f) /* Endpoint ID 1 - 31 */
819 #define TRB_TO_EP_INDEX(p)	(TRB_TO_EP_ID(p) - 1) /* Endpoint index 0 - 30 */
820 #define EP_INDEX_FOR_TRB(p)	((((p) + 1) & 0x1f) << 16)
825 /* Completion Code - only applicable for some types of TRBs */
829 #define COMP_SUCCESS				1
923 		return "Stopped - Length Invalid";  in xhci_trb_comp_code_string()
925 		return "Stopped - Short Packet";  in xhci_trb_comp_code_string()
946 	/* 64-bit segment pointer*/
953 #define LINK_TOGGLE	(0x1<<1)
966 /* Address device - disable SetAddress */
967 #define TRB_BSR		(1<<9)
969 /* Configure Endpoint - Deconfigure */
970 #define TRB_DC		(1<<9)
972 /* Stop Ring - Transfer State Preserve */
973 #define TRB_TSP		(1<<9)
1002 /* Stop Endpoint TRB - ep_index to endpoint ID for this TRB */
1003 #define SUSPEND_PORT_FOR_TRB(p)		(((p) & 1) << 23)
1004 #define TRB_TO_SUSPEND_PORT(p)		(((p) & (1 << 23)) >> 23)
1010 #define SCT_FOR_TRB(p)			(((p) & 0x7) << 1)
1013 #define TRB_TC			(1<<1)
1016 /* Port ID - bits 31:24 */
1019 #define EVENT_DATA		(1 << 2)
1022 /* transfer_len bitmasks - bits 0:16 */
1027 /* xhci 1.1 uses the TD_SIZE field for TBC if Extended TBC is enabled (ETE) */
1029 /* Interrupter Target - which MSI-X vector to target the completion event at */
1033 /* Cycle bit - indicates TRB ownership by HC or HCD */
1034 #define TRB_CYCLE		(1<<0)
1039 #define TRB_ENT			(1<<1)
1041 #define TRB_ISP			(1<<2)
1043 #define TRB_NO_SNOOP		(1<<3)
1045 #define TRB_CHAIN		(1<<4)
1047 #define TRB_IOC			(1<<5)
1049 #define TRB_IDT			(1<<6)
1054 #define	TRB_BEI			(1<<9)
1057 #define TRB_DIR_IN		(1<<16)
1063 #define TRB_SIA			(1<<31)
1066 /* Total burst count field, Rsvdz on xhci 1.1 with Extended TBC enabled (ETE) */
1093 #define TRB_NORMAL		1
1105 /* Transfer Ring No-op (not for the command ring) */
1134 /* Force Header Command - generate a transaction or link management packet */
1136 /* No-op Command - not for transfer rings */
1138 /* TRB IDs 24-31 reserved */
1152 /* Device Notification Event - device sent function wake notification */
1154 /* MFINDEX Wrap Event - microframe counter wrapped */
1156 /* TRB IDs 40-47 reserved, 48-63 is vendor-defined */
1158 /* Nec vendor-specific command completion event. */
1181 		return "No-Op";  in xhci_trb_type_string()
1211 		return "No-Op Command";  in xhci_trb_type_string()
1238 /* Above, but for __le32 types -- can avoid work by swapping constants: */
1249  * since the command ring is 64-byte aligned.
1254 #define MAX_RSVD_CMD_TRBS	(TRBS_PER_SEGMENT - 3)
1259 #define TRB_MAX_BUFF_SIZE	(1 << TRB_MAX_BUFF_SHIFT)
1261 #define TRB_BUFF_LEN_UP_TO_BOUNDARY(addr)	(TRB_MAX_BUFF_SIZE - \
1262 					(addr & (TRB_MAX_BUFF_SIZE - 1)))
1272 	for (seg = head; seg != NULL; seg = (seg->next != head ? seg->next : NULL))
1280 	/* Max packet sized bounce buffer for td-fragmant alignment */
1366 	 * if we own the TRB (if we are the consumer).  See section 4.9.1.
1379 	/* 64-bit event ring segment address */
1389 	/* xhci->event_ring keeps track of segment dma addresses */
1431 	/* ports suspend status arrays - max 31 ports for USB2, 15 for USB3 */
1473 	unsigned int		lpm_incapable:1;
1503 	/* Cached register copies of read-only HC data */
1521 	/* MSI-X/MSI vectors */
1533 #define CMD_RING_STATE_RUNNING         (1 << 0)
1534 #define CMD_RING_STATE_ABORTED         (1 << 1)
1535 #define CMD_RING_STATE_STOPPED         (1 << 2)
1563 /* Host controller is dying - not responding to commands. "I'm not dead yet!"
1566  * halt the xHCI host, and complete all URBs with an -ESHUTDOWN code.  Any code
1569  * they see this status (any time they drop and re-acquire xhci->lock).
1575 #define XHCI_STATE_DYING	(1 << 0)
1576 #define XHCI_STATE_HALTED	(1 << 1)
1577 #define XHCI_STATE_REMOVING	(1 << 2)
1580 #define XHCI_RESET_EP_QUIRK	BIT_ULL(1) /* Deprecated */
1647 	unsigned		hw_lpm_support:1;
1649 	unsigned		broken_suspend:1;
1651 	unsigned		allow_single_roothub:1;
1667 	/* platform-specific data -- must come last */
1698 		primary_hcd = hcd->primary_hcd;  in hcd_to_xhci()
1700 	return (struct xhci_hcd *) (primary_hcd->hcd_priv);  in hcd_to_xhci()
1705 	return xhci->main_hcd;  in xhci_to_hcd()
1710 	if (xhci->shared_hcd)  in xhci_get_usb3_hcd()
1711 		return xhci->shared_hcd;  in xhci_get_usb3_hcd()
1713 	if (!xhci->usb2_rhub.num_ports)  in xhci_get_usb3_hcd()
1714 		return xhci->main_hcd;  in xhci_get_usb3_hcd()
1728 	return xhci->allow_single_roothub &&  in xhci_has_one_roothub()
1729 	       (!xhci->usb2_rhub.num_ports || !xhci->usb3_rhub.num_ports);  in xhci_has_one_roothub()
1733 	dev_dbg(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1735 	dev_err(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1737 	dev_warn(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1739 	dev_info(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1744  * Some xHCI implementations may support 64-bit address pointers.  Registers
1745  * with 64-bit address pointers should be written to with dword accesses by
1746  * writing the low dword first (ptr[0]), then the high dword (ptr[1]) second.
1747  * xHCI implementations that do not support 64-bit address pointers will ignore
1774 	return (xhci->quirks & XHCI_LINK_TRB_QUIRK) ||  in xhci_link_chain_quirk()
1775 	       (type == TYPE_ISOC && (xhci->quirks & (XHCI_AMD_0x96_HOST | XHCI_NEC_HOST)));  in xhci_link_chain_quirk()
1979 	return xhci_triad_to_transfer_ring(xhci, urb->dev->slot_id,  in xhci_urb_to_transfer_ring()
1980 					xhci_get_endpoint_index(&urb->ep->desc),  in xhci_urb_to_transfer_ring()
1981 					urb->stream_id);  in xhci_urb_to_transfer_ring()
1991 	if (!usb_endpoint_xfer_isoc(&urb->ep->desc) && usb_urb_dir_out(urb) &&  in xhci_urb_suitable_for_idt()
1992 	    usb_endpoint_maxp(&urb->ep->desc) >= TRB_IDT_MAX_SIZE &&  in xhci_urb_suitable_for_idt()
1993 	    urb->transfer_buffer_length <= TRB_IDT_MAX_SIZE &&  in xhci_urb_suitable_for_idt()
1994 	    !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) &&  in xhci_urb_suitable_for_idt()
1995 	    !urb->num_sgs)  in xhci_urb_suitable_for_idt()
2244 			"type '%s' -> raw %08x %08x %08x %08x",  in xhci_decode_trb()
2299 				s = "full-speed";  in xhci_decode_slot_context()
2302 				s = "low-speed";  in xhci_decode_slot_context()
2305 				s = "high-speed";  in xhci_decode_slot_context()
2308 				s = "super-speed";  in xhci_decode_slot_context()
2311 				s = "super-speed plus";  in xhci_decode_slot_context()
2316 			mtt ? " multi-TT" : "",  in xhci_decode_slot_context()
2378 		      portsc & PORT_POWER	? "Powered" : "Powered-off",  in xhci_decode_portsc()
2379 		      portsc & PORT_CONNECT	? "Connected" : "Not-connected",  in xhci_decode_portsc()
2387 		ret += sprintf(str + ret, "In-Reset ");  in xhci_decode_portsc()
2543 	mult = CTX_TO_EP_MULT(info) + 1;  in xhci_decode_ep_context()
2546 	cerr = (info2 & (3 << 1)) >> 1;  in xhci_decode_ep_context()
2548 	hid = !!(info2 & (1 << 7));  in xhci_decode_ep_context()
2559 			(1 << interval) * 125, esit, cerr);  in xhci_decode_ep_context()