1 // Copyright 2018 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #![allow(clippy::result_large_err)] 6 7 mod command_ring_controller; 8 mod device_slot; 9 mod event_ring; 10 mod interrupter; 11 mod intr_resample_handler; 12 mod ring_buffer; 13 mod ring_buffer_controller; 14 mod ring_buffer_stop_cb; 15 mod transfer_ring_controller; 16 #[allow(dead_code)] 17 mod xhci_abi; 18 #[allow(dead_code)] 19 mod xhci_regs; 20 21 pub mod scatter_gather_buffer; 22 pub mod usb_hub; 23 pub mod xhci_backend_device; 24 pub mod xhci_backend_device_provider; 25 pub mod xhci_controller; 26 pub mod xhci_transfer; 27 28 use std::sync::Arc; 29 use std::thread; 30 31 use base::debug; 32 use base::error; 33 use remain::sorted; 34 use sync::Mutex; 35 use thiserror::Error; 36 use vm_memory::GuestAddress; 37 use vm_memory::GuestMemory; 38 39 use crate::usb::backend::error::Error as BackendProviderError; 40 use crate::usb::xhci::command_ring_controller::CommandRingController; 41 use crate::usb::xhci::command_ring_controller::CommandRingControllerError; 42 use crate::usb::xhci::device_slot::DeviceSlots; 43 use crate::usb::xhci::device_slot::Error as DeviceSlotError; 44 use crate::usb::xhci::interrupter::Error as InterrupterError; 45 use crate::usb::xhci::interrupter::Interrupter; 46 use crate::usb::xhci::intr_resample_handler::IntrResampleHandler; 47 use crate::usb::xhci::ring_buffer_stop_cb::RingBufferStopCallback; 48 use crate::usb::xhci::usb_hub::UsbHub; 49 use crate::usb::xhci::xhci_backend_device_provider::XhciBackendDeviceProvider; 50 use crate::usb::xhci::xhci_regs::*; 51 use crate::utils::Error as UtilsError; 52 use crate::utils::EventLoop; 53 use crate::utils::FailHandle; 54 use crate::IrqLevelEvent; 55 56 #[sorted] 57 #[derive(Error, Debug)] 58 pub enum Error { 59 #[error("failed to clone irq event: {0}")] 60 CloneIrqEvent(base::Error), 61 #[error("failed to clone resample event: {0}")] 62 CloneResampleEvent(base::Error), 63 #[error("failed to create command ring controller: {0}")] 64 CreateCommandRingController(CommandRingControllerError), 65 #[error("failed to enable interrupter: {0}")] 66 EnableInterrupter(InterrupterError), 67 #[error("failed to get device slot: {0}")] 68 GetDeviceSlot(u8), 69 #[error("failed to reset port")] 70 ResetPort, 71 #[error("failed to ring doorbell: {0}")] 72 RingDoorbell(DeviceSlotError), 73 #[error("failed to send interrupt: {0}")] 74 SendInterrupt(InterrupterError), 75 #[error("failed to set interrupter moderation: {0}")] 76 SetModeration(InterrupterError), 77 #[error("failed to setup event ring and event handler busy: {0}")] 78 SetupEventRing(InterrupterError), 79 #[error("failed to start event loop: {0}")] 80 StartEventLoop(UtilsError), 81 #[error("failed to start backend provider: {0}")] 82 StartProvider(BackendProviderError), 83 #[error("failed to start resample handler")] 84 StartResampleHandler, 85 } 86 87 type Result<T> = std::result::Result<T, Error>; 88 89 /// xHCI controller implementation. 90 pub struct Xhci { 91 fail_handle: Arc<dyn FailHandle>, 92 regs: XhciRegs, 93 interrupter: Arc<Mutex<Interrupter>>, 94 command_ring_controller: Arc<CommandRingController>, 95 device_slots: DeviceSlots, 96 event_loop: Arc<EventLoop>, 97 event_loop_join_handle: Option<thread::JoinHandle<()>>, 98 // resample handler and device provider only lives on EventLoop to handle corresponding events. 99 // By design, event loop only hold weak reference. We need to keep a strong reference here to 100 // keep it alive. 101 #[allow(dead_code)] 102 intr_resample_handler: Arc<IntrResampleHandler>, 103 #[allow(dead_code)] 104 device_provider: Box<dyn XhciBackendDeviceProvider>, 105 } 106 107 impl Xhci { 108 /// Create a new xHCI controller. new( fail_handle: Arc<dyn FailHandle>, mem: GuestMemory, device_provider: Box<dyn XhciBackendDeviceProvider>, interrupt_evt: IrqLevelEvent, regs: XhciRegs, ) -> Result<Arc<Self>>109 pub fn new( 110 fail_handle: Arc<dyn FailHandle>, 111 mem: GuestMemory, 112 device_provider: Box<dyn XhciBackendDeviceProvider>, 113 interrupt_evt: IrqLevelEvent, 114 regs: XhciRegs, 115 ) -> Result<Arc<Self>> { 116 let (event_loop, join_handle) = 117 EventLoop::start("xhci".to_string(), Some(fail_handle.clone())) 118 .map_err(Error::StartEventLoop)?; 119 let irq_evt = interrupt_evt 120 .get_trigger() 121 .try_clone() 122 .map_err(Error::CloneIrqEvent)?; 123 let interrupter = Arc::new(Mutex::new(Interrupter::new(mem.clone(), irq_evt, ®s))); 124 let event_loop = Arc::new(event_loop); 125 let irq_resample_evt = interrupt_evt 126 .get_resample() 127 .try_clone() 128 .map_err(Error::CloneResampleEvent)?; 129 let intr_resample_handler = 130 IntrResampleHandler::start(&event_loop, interrupter.clone(), irq_resample_evt) 131 .ok_or(Error::StartResampleHandler)?; 132 let hub = Arc::new(UsbHub::new(®s, interrupter.clone())); 133 134 let mut device_provider = device_provider; 135 device_provider 136 .start(fail_handle.clone(), event_loop.clone(), hub.clone()) 137 .map_err(Error::StartProvider)?; 138 139 let device_slots = DeviceSlots::new( 140 fail_handle.clone(), 141 regs.dcbaap.clone(), 142 hub, 143 interrupter.clone(), 144 event_loop.clone(), 145 mem.clone(), 146 ); 147 let command_ring_controller = CommandRingController::new( 148 mem, 149 event_loop.clone(), 150 device_slots.clone(), 151 interrupter.clone(), 152 ) 153 .map_err(Error::CreateCommandRingController)?; 154 let xhci = Arc::new(Xhci { 155 fail_handle, 156 regs, 157 intr_resample_handler, 158 interrupter, 159 command_ring_controller, 160 device_slots, 161 device_provider, 162 event_loop, 163 event_loop_join_handle: Some(join_handle), 164 }); 165 Self::init_reg_callbacks(&xhci); 166 Ok(xhci) 167 } 168 init_reg_callbacks(xhci: &Arc<Xhci>)169 fn init_reg_callbacks(xhci: &Arc<Xhci>) { 170 // All the callbacks will hold a weak reference to avoid memory leak. Thos weak upgrade 171 // should never fail. 172 let xhci_weak = Arc::downgrade(xhci); 173 xhci.regs.usbcmd.set_write_cb(move |val: u32| { 174 // All the weak reference upgrade should never fail. xhci hold reference to the 175 // registers, callback won't be invoked if xhci is gone. 176 let xhci = xhci_weak.upgrade().unwrap(); 177 let r = xhci.usbcmd_callback(val); 178 xhci.handle_register_callback_result(r, 0) 179 }); 180 181 let xhci_weak = Arc::downgrade(xhci); 182 xhci.regs.crcr.set_write_cb(move |val: u64| { 183 let xhci = xhci_weak.upgrade().unwrap(); 184 xhci.crcr_callback(val) 185 }); 186 187 for i in 0..xhci.regs.portsc.len() { 188 let xhci_weak = Arc::downgrade(xhci); 189 xhci.regs.portsc[i].set_write_cb(move |val: u32| { 190 let xhci = xhci_weak.upgrade().unwrap(); 191 let r = xhci.portsc_callback(i as u32, val); 192 xhci.handle_register_callback_result(r, 0) 193 }); 194 } 195 196 for i in 0..xhci.regs.doorbells.len() { 197 let xhci_weak = Arc::downgrade(xhci); 198 xhci.regs.doorbells[i].set_write_cb(move |val: u32| { 199 let xhci = xhci_weak.upgrade().unwrap(); 200 let r = xhci.doorbell_callback(i as u32, val); 201 xhci.handle_register_callback_result(r, ()); 202 val 203 }); 204 } 205 206 let xhci_weak = Arc::downgrade(xhci); 207 xhci.regs.iman.set_write_cb(move |val: u32| { 208 let xhci = xhci_weak.upgrade().unwrap(); 209 let r = xhci.iman_callback(val); 210 xhci.handle_register_callback_result(r, ()); 211 val 212 }); 213 214 let xhci_weak = Arc::downgrade(xhci); 215 xhci.regs.imod.set_write_cb(move |val: u32| { 216 let xhci = xhci_weak.upgrade().unwrap(); 217 let r = xhci.imod_callback(val); 218 xhci.handle_register_callback_result(r, ()); 219 val 220 }); 221 222 let xhci_weak = Arc::downgrade(xhci); 223 xhci.regs.erstsz.set_write_cb(move |val: u32| { 224 let xhci = xhci_weak.upgrade().unwrap(); 225 let r = xhci.erstsz_callback(val); 226 xhci.handle_register_callback_result(r, ()); 227 val 228 }); 229 230 let xhci_weak = Arc::downgrade(xhci); 231 xhci.regs.erstba.set_write_cb(move |val: u64| { 232 let xhci = xhci_weak.upgrade().unwrap(); 233 let r = xhci.erstba_callback(val); 234 xhci.handle_register_callback_result(r, ()); 235 val 236 }); 237 238 let xhci_weak = Arc::downgrade(xhci); 239 xhci.regs.erdp.set_write_cb(move |val: u64| { 240 let xhci = xhci_weak.upgrade().unwrap(); 241 let r = xhci.erdp_callback(val); 242 xhci.handle_register_callback_result(r, ()); 243 val 244 }); 245 } 246 handle_register_callback_result<T>(&self, r: Result<T>, t: T) -> T247 fn handle_register_callback_result<T>(&self, r: Result<T>, t: T) -> T { 248 match r { 249 Ok(v) => v, 250 Err(e) => { 251 error!("xhci controller failed: {}", e); 252 self.fail_handle.fail(); 253 t 254 } 255 } 256 } 257 258 // Callback for usbcmd register write. usbcmd_callback(&self, value: u32) -> Result<u32>259 fn usbcmd_callback(&self, value: u32) -> Result<u32> { 260 if (value & USB_CMD_RESET) > 0 { 261 debug!("xhci_controller: reset controller"); 262 self.reset(); 263 return Ok(value & (!USB_CMD_RESET)); 264 } 265 266 if (value & USB_CMD_RUNSTOP) > 0 { 267 debug!("xhci_controller: clear halt bits"); 268 self.regs.usbsts.clear_bits(USB_STS_HALTED); 269 } else { 270 debug!("xhci_controller: halt device"); 271 self.halt(); 272 self.regs.crcr.clear_bits(CRCR_COMMAND_RING_RUNNING); 273 } 274 275 // Enable interrupter if needed. 276 let enabled = (value & USB_CMD_INTERRUPTER_ENABLE) > 0 277 && (self.regs.iman.get_value() & IMAN_INTERRUPT_ENABLE) > 0; 278 debug!("xhci_controller: interrupter enable?: {}", enabled); 279 self.interrupter 280 .lock() 281 .set_enabled(enabled) 282 .map_err(Error::EnableInterrupter)?; 283 Ok(value) 284 } 285 286 // Callback for crcr register write. crcr_callback(&self, value: u64) -> u64287 fn crcr_callback(&self, value: u64) -> u64 { 288 let _trace = cros_tracing::trace_event!(USB, "crcr_callback", value); 289 if (self.regs.crcr.get_value() & CRCR_COMMAND_RING_RUNNING) == 0 { 290 self.command_ring_controller 291 .set_dequeue_pointer(GuestAddress(value & CRCR_COMMAND_RING_POINTER)); 292 self.command_ring_controller 293 .set_consumer_cycle_state((value & CRCR_RING_CYCLE_STATE) > 0); 294 value 295 } else { 296 error!("Write to crcr while command ring is running"); 297 self.regs.crcr.get_value() 298 } 299 } 300 301 // Callback for portsc register write. portsc_callback(&self, index: u32, value: u32) -> Result<u32>302 fn portsc_callback(&self, index: u32, value: u32) -> Result<u32> { 303 let _trace = cros_tracing::trace_event!(USB, "portsc_callback", index, value); 304 let mut value = value; 305 let port_id = (index + 1) as u8; 306 // xHCI spec 4.19.5. Note: we might want to change this logic if we support USB 3.0. 307 if (value & PORTSC_PORT_RESET) > 0 || (value & PORTSC_WARM_PORT_RESET) > 0 { 308 self.device_slots 309 .reset_port(port_id) 310 .map_err(|_| Error::ResetPort)?; 311 value &= !PORTSC_PORT_LINK_STATE_MASK; 312 value &= !PORTSC_PORT_RESET; 313 value |= PORTSC_PORT_ENABLED; 314 value |= PORTSC_PORT_RESET_CHANGE; 315 self.interrupter 316 .lock() 317 .send_port_status_change_trb(port_id) 318 .map_err(Error::SendInterrupt)?; 319 } 320 Ok(value) 321 } 322 323 // Callback for doorbell register write. doorbell_callback(&self, index: u32, value: u32) -> Result<()>324 fn doorbell_callback(&self, index: u32, value: u32) -> Result<()> { 325 let _trace = cros_tracing::trace_event!(USB, "doorbell_callback", index, value); 326 let target = (value & DOORBELL_TARGET) as u8; 327 let stream_id: u16 = (value >> DOORBELL_STREAM_ID_OFFSET) as u16; 328 if (self.regs.usbcmd.get_value() & USB_CMD_RUNSTOP) > 0 { 329 // First doorbell is for command ring. 330 if index == 0 { 331 if target != 0 || stream_id != 0 { 332 return Ok(()); 333 } 334 self.regs.crcr.set_bits(CRCR_COMMAND_RING_RUNNING); 335 self.command_ring_controller.start(); 336 } else { 337 self.device_slots 338 .slot(index as u8) 339 .ok_or(Error::GetDeviceSlot(index as u8))? 340 .ring_doorbell(target, stream_id) 341 .map_err(Error::RingDoorbell)?; 342 } 343 } 344 Ok(()) 345 } 346 347 // Callback for iman register write. iman_callback(&self, value: u32) -> Result<()>348 fn iman_callback(&self, value: u32) -> Result<()> { 349 let _trace = cros_tracing::trace_event!(USB, "iman_callback", value); 350 let enabled = ((value & IMAN_INTERRUPT_ENABLE) > 0) 351 && ((self.regs.usbcmd.get_value() & USB_CMD_INTERRUPTER_ENABLE) > 0); 352 self.interrupter 353 .lock() 354 .set_enabled(enabled) 355 .map_err(Error::EnableInterrupter) 356 } 357 358 // Callback for imod register write. imod_callback(&self, value: u32) -> Result<()>359 fn imod_callback(&self, value: u32) -> Result<()> { 360 let _trace = cros_tracing::trace_event!(USB, "imod_callback", value); 361 self.interrupter 362 .lock() 363 .set_moderation( 364 (value & IMOD_INTERRUPT_MODERATION_INTERVAL) as u16, 365 (value >> IMOD_INTERRUPT_MODERATION_COUNTER_OFFSET) as u16, 366 ) 367 .map_err(Error::SetModeration) 368 } 369 370 // Callback for erstsz register write. erstsz_callback(&self, value: u32) -> Result<()>371 fn erstsz_callback(&self, value: u32) -> Result<()> { 372 let _trace = cros_tracing::trace_event!(USB, "erstsz_callback", value); 373 self.interrupter 374 .lock() 375 .set_event_ring_seg_table_size((value & ERSTSZ_SEGMENT_TABLE_SIZE) as u16) 376 .map_err(Error::SetupEventRing) 377 } 378 379 // Callback for erstba register write. erstba_callback(&self, value: u64) -> Result<()>380 fn erstba_callback(&self, value: u64) -> Result<()> { 381 let _trace = cros_tracing::trace_event!(USB, "erstba_callback", value); 382 self.interrupter 383 .lock() 384 .set_event_ring_seg_table_base_addr(GuestAddress( 385 value & ERSTBA_SEGMENT_TABLE_BASE_ADDRESS, 386 )) 387 .map_err(Error::SetupEventRing) 388 } 389 390 // Callback for erdp register write. erdp_callback(&self, value: u64) -> Result<()>391 fn erdp_callback(&self, value: u64) -> Result<()> { 392 let _trace = cros_tracing::trace_event!(USB, "erdp_callback", value); 393 self.interrupter 394 .lock() 395 .set_event_ring_dequeue_pointer( 396 GuestAddress(value & ERDP_EVENT_RING_DEQUEUE_POINTER), 397 (value & ERDP_EVENT_HANDLER_BUSY) > 0, 398 ) 399 .map_err(Error::SetupEventRing) 400 } 401 reset(&self)402 fn reset(&self) { 403 self.regs.usbsts.set_bits(USB_STS_CONTROLLER_NOT_READY); 404 let usbsts = self.regs.usbsts.clone(); 405 self.device_slots.stop_all_and_reset(move || { 406 usbsts.clear_bits(USB_STS_CONTROLLER_NOT_READY); 407 }); 408 } 409 halt(&self)410 fn halt(&self) { 411 let usbsts = self.regs.usbsts.clone(); 412 self.device_slots 413 .stop_all(RingBufferStopCallback::new(move || { 414 usbsts.set_bits(USB_STS_HALTED); 415 })); 416 } 417 } 418 419 impl Drop for Xhci { drop(&mut self)420 fn drop(&mut self) { 421 self.event_loop.stop(); 422 if let Some(join_handle) = self.event_loop_join_handle.take() { 423 let _ = join_handle.join(); 424 } 425 } 426 } 427