1 // Copyright 2022 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #![cfg_attr(windows, allow(dead_code))] 6 7 use base::error; 8 use base::AsRawDescriptor; 9 use base::Event; 10 use base::RawDescriptor; 11 use base::Tube; 12 use bit_field::*; 13 use vm_control::VmIrqRequest; 14 use vm_control::VmIrqResponse; 15 use zerocopy::AsBytes; 16 use zerocopy::FromBytes; 17 use zerocopy::FromZeroes; 18 19 use crate::pci::pci_configuration::PciCapConfig; 20 use crate::pci::pci_configuration::PciCapConfigWriteResult; 21 use crate::pci::PciCapability; 22 use crate::pci::PciCapabilityID; 23 24 // MSI registers 25 pub const PCI_MSI_NEXT_POINTER: u32 = 0x1; // Next cap pointer 26 pub const PCI_MSI_FLAGS: u32 = 0x2; // Message Control 27 const PCI_MSI_FLAGS_ENABLE: u16 = 0x0001; // MSI feature enabled 28 pub const PCI_MSI_FLAGS_64BIT: u16 = 0x0080; // 64-bit addresses allowed 29 pub const PCI_MSI_FLAGS_MASKBIT: u16 = 0x0100; // Per-vector masking capable 30 const PCI_MSI_ADDRESS_LO: u32 = 0x4; // MSI address lower 32 bits 31 const PCI_MSI_ADDRESS_HI: u32 = 0x8; // MSI address upper 32 bits (if 64 bit allowed) 32 const PCI_MSI_DATA_32: u32 = 0x8; // 16 bits of data for 32-bit message address 33 const PCI_MSI_DATA_64: u32 = 0xC; // 16 bits of date for 64-bit message address 34 35 // MSI length 36 const MSI_LENGTH_32BIT_WITHOUT_MASK: u32 = 0xA; 37 const MSI_LENGTH_32BIT_WITH_MASK: u32 = 0x14; 38 const MSI_LENGTH_64BIT_WITHOUT_MASK: u32 = 0xE; 39 const MSI_LENGTH_64BIT_WITH_MASK: u32 = 0x18; 40 41 pub enum MsiStatus { 42 Enabled, 43 Disabled, 44 NothingToDo, 45 } 46 47 /// Wrapper over MSI Capability Structure 48 pub struct MsiConfig { 49 is_64bit: bool, 50 mask_cap: bool, 51 ctrl: u16, 52 address: u64, 53 data: u16, 54 vm_socket_irq: Tube, 55 irqfd: Option<Event>, 56 gsi: Option<u32>, 57 device_id: u32, 58 device_name: String, 59 } 60 61 impl MsiConfig { new( is_64bit: bool, mask_cap: bool, vm_socket_irq: Tube, device_id: u32, device_name: String, ) -> Self62 pub fn new( 63 is_64bit: bool, 64 mask_cap: bool, 65 vm_socket_irq: Tube, 66 device_id: u32, 67 device_name: String, 68 ) -> Self { 69 let mut ctrl: u16 = 0; 70 if is_64bit { 71 ctrl |= PCI_MSI_FLAGS_64BIT; 72 } 73 if mask_cap { 74 ctrl |= PCI_MSI_FLAGS_MASKBIT; 75 } 76 MsiConfig { 77 is_64bit, 78 mask_cap, 79 ctrl, 80 address: 0, 81 data: 0, 82 vm_socket_irq, 83 irqfd: None, 84 gsi: None, 85 device_id, 86 device_name, 87 } 88 } 89 len(&self) -> u3290 fn len(&self) -> u32 { 91 match (self.is_64bit, self.mask_cap) { 92 (true, true) => MSI_LENGTH_64BIT_WITH_MASK, 93 (true, false) => MSI_LENGTH_64BIT_WITHOUT_MASK, 94 (false, true) => MSI_LENGTH_32BIT_WITH_MASK, 95 (false, false) => MSI_LENGTH_32BIT_WITHOUT_MASK, 96 } 97 } 98 is_msi_reg(&self, offset: u32, index: u64, len: usize) -> bool99 pub fn is_msi_reg(&self, offset: u32, index: u64, len: usize) -> bool { 100 let msi_len = self.len(); 101 index >= offset as u64 102 && index + len as u64 <= (offset + msi_len) as u64 103 && len as u32 <= msi_len 104 } 105 read_msi_capability(&self, offset: u32, data: u32) -> u32106 pub fn read_msi_capability(&self, offset: u32, data: u32) -> u32 { 107 if offset == 0 { 108 (self.ctrl as u32) << 16 | (data & u16::MAX as u32) 109 } else { 110 data 111 } 112 } 113 write_msi_capability(&mut self, offset: u32, data: &[u8]) -> MsiStatus114 pub fn write_msi_capability(&mut self, offset: u32, data: &[u8]) -> MsiStatus { 115 let len = data.len(); 116 let mut ret = MsiStatus::NothingToDo; 117 let old_address = self.address; 118 let old_data = self.data; 119 120 // write msi ctl 121 if len == 2 && offset == PCI_MSI_FLAGS { 122 let was_enabled = self.is_msi_enabled(); 123 let value: [u8; 2] = [data[0], data[1]]; 124 self.ctrl = u16::from_le_bytes(value); 125 let is_enabled = self.is_msi_enabled(); 126 if !was_enabled && is_enabled { 127 self.enable(); 128 ret = MsiStatus::Enabled; 129 } else if was_enabled && !is_enabled { 130 ret = MsiStatus::Disabled; 131 } 132 } else if len == 4 && offset == PCI_MSI_ADDRESS_LO && !self.is_64bit { 133 //write 32 bit message address 134 let value: [u8; 8] = [data[0], data[1], data[2], data[3], 0, 0, 0, 0]; 135 self.address = u64::from_le_bytes(value); 136 } else if len == 4 && offset == PCI_MSI_ADDRESS_LO && self.is_64bit { 137 // write 64 bit message address low part 138 let value: [u8; 8] = [data[0], data[1], data[2], data[3], 0, 0, 0, 0]; 139 self.address &= !0xffffffff; 140 self.address |= u64::from_le_bytes(value); 141 } else if len == 4 && offset == PCI_MSI_ADDRESS_HI && self.is_64bit { 142 //write 64 bit message address high part 143 let value: [u8; 8] = [0, 0, 0, 0, data[0], data[1], data[2], data[3]]; 144 self.address &= 0xffffffff; 145 self.address |= u64::from_le_bytes(value); 146 } else if len == 8 && offset == PCI_MSI_ADDRESS_LO && self.is_64bit { 147 // write 64 bit message address 148 let value: [u8; 8] = [ 149 data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], 150 ]; 151 self.address = u64::from_le_bytes(value); 152 } else if len == 2 153 && ((offset == PCI_MSI_DATA_32 && !self.is_64bit) 154 || (offset == PCI_MSI_DATA_64 && self.is_64bit)) 155 { 156 // write message data 157 let value: [u8; 2] = [data[0], data[1]]; 158 self.data = u16::from_le_bytes(value); 159 } 160 161 if self.is_msi_enabled() && (old_address != self.address || old_data != self.data) { 162 self.add_msi_route(); 163 } 164 165 ret 166 } 167 is_msi_enabled(&self) -> bool168 pub fn is_msi_enabled(&self) -> bool { 169 self.ctrl & PCI_MSI_FLAGS_ENABLE == PCI_MSI_FLAGS_ENABLE 170 } 171 add_msi_route(&self)172 fn add_msi_route(&self) { 173 let gsi = match self.gsi { 174 Some(g) => g, 175 None => { 176 error!("Add msi route but gsi is none"); 177 return; 178 } 179 }; 180 if let Err(e) = self.vm_socket_irq.send(&VmIrqRequest::AddMsiRoute { 181 gsi, 182 msi_address: self.address, 183 msi_data: self.data.into(), 184 }) { 185 error!("failed to send AddMsiRoute request at {:?}", e); 186 return; 187 } 188 match self.vm_socket_irq.recv() { 189 Ok(VmIrqResponse::Err(e)) => error!("failed to call AddMsiRoute request {:?}", e), 190 Ok(_) => {} 191 Err(e) => error!("failed to receive AddMsiRoute response {:?}", e), 192 } 193 } 194 allocate_one_msi(&mut self)195 fn allocate_one_msi(&mut self) { 196 let irqfd = match self.irqfd.take() { 197 Some(e) => e, 198 None => match Event::new() { 199 Ok(e) => e, 200 Err(e) => { 201 error!("failed to create event: {:?}", e); 202 return; 203 } 204 }, 205 }; 206 207 let request = VmIrqRequest::AllocateOneMsi { 208 irqfd, 209 device_id: self.device_id, 210 queue_id: 0, 211 device_name: self.device_name.clone(), 212 }; 213 let request_result = self.vm_socket_irq.send(&request); 214 215 // Stash the irqfd in self immediately because we used take above. 216 self.irqfd = match request { 217 VmIrqRequest::AllocateOneMsi { irqfd, .. } => Some(irqfd), 218 _ => unreachable!(), 219 }; 220 221 if let Err(e) = request_result { 222 error!("failed to send AllocateOneMsi request: {:?}", e); 223 return; 224 } 225 226 match self.vm_socket_irq.recv() { 227 Ok(VmIrqResponse::AllocateOneMsi { gsi }) => self.gsi = Some(gsi), 228 _ => error!("failed to receive AllocateOneMsi Response"), 229 } 230 } 231 enable(&mut self)232 fn enable(&mut self) { 233 if self.gsi.is_none() || self.irqfd.is_none() { 234 self.allocate_one_msi(); 235 } 236 237 self.add_msi_route(); 238 } 239 get_irqfd(&self) -> Option<&Event>240 pub fn get_irqfd(&self) -> Option<&Event> { 241 self.irqfd.as_ref() 242 } 243 destroy(&mut self)244 pub fn destroy(&mut self) { 245 if let Some(gsi) = self.gsi { 246 if let Some(irqfd) = self.irqfd.take() { 247 let request = VmIrqRequest::ReleaseOneIrq { gsi, irqfd }; 248 if self.vm_socket_irq.send(&request).is_ok() { 249 let _ = self.vm_socket_irq.recv::<VmIrqResponse>(); 250 } 251 } 252 } 253 } 254 255 /// Return the raw descriptor of the MSI device socket get_msi_socket(&self) -> RawDescriptor256 pub fn get_msi_socket(&self) -> RawDescriptor { 257 self.vm_socket_irq.as_raw_descriptor() 258 } 259 trigger(&self)260 pub fn trigger(&self) { 261 if let Some(irqfd) = self.irqfd.as_ref() { 262 irqfd.signal().unwrap(); 263 } 264 } 265 } 266 267 #[bitfield] 268 #[derive(Copy, Clone, AsBytes, FromZeroes, FromBytes)] 269 pub struct MsiCtrl { 270 enable: B1, 271 multi_msg_capable: B3, 272 multi_msg_enable: B3, 273 is_64bit: B1, 274 per_vector_masking: B1, 275 extended_msg_data_capable: B1, 276 extended_msg_data_enable: B1, 277 reserved: B5, 278 } 279 280 #[allow(dead_code)] 281 #[repr(C, align(4))] 282 #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] 283 struct Msi32BitWithoutMask { 284 msg_data: u16, 285 msg_extended_data: u16, 286 _padding: [u8; 12], 287 } 288 289 #[allow(dead_code)] 290 #[repr(C, align(4))] 291 #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] 292 struct Msi32BitWithMask { 293 msg_data: u16, 294 msg_extended_data: u16, 295 mask_bits: u32, 296 pending_bits: u32, 297 _padding: [u8; 4], 298 } 299 300 #[allow(dead_code)] 301 #[repr(C, align(4))] 302 #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] 303 struct Msi64BitWithoutMask { 304 msg_upper: u32, 305 msg_data: u16, 306 msg_extended_data: u16, 307 _padding: [u8; 8], 308 } 309 310 #[allow(dead_code)] 311 #[repr(C)] 312 #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] 313 struct Msi64BitWithMask { 314 msg_upper: u32, 315 msg_data: u16, 316 msg_extended_data: u16, 317 mask_bits: u32, 318 pending_bits: u32, 319 } 320 321 #[allow(dead_code)] 322 #[repr(C)] 323 #[derive(Clone, Copy, AsBytes, FromZeroes, FromBytes)] 324 union MsiVary { 325 msi_32bit_without_mask: Msi32BitWithoutMask, 326 msi_32bit_with_mask: Msi32BitWithMask, 327 msi_64bit_without_mask: Msi64BitWithoutMask, 328 msi_64bit_with_mask: Msi64BitWithMask, 329 } 330 331 #[allow(dead_code)] 332 #[repr(C)] 333 #[derive(Clone, Copy, AsBytes, FromZeroes, FromBytes)] 334 /// MSI Capability Structure 335 pub struct MsiCap { 336 // To make add_capability() happy 337 _cap_vndr: u8, 338 _cap_next: u8, 339 // Message Control Register 340 msg_ctl: MsiCtrl, 341 // Message Address 342 msg_addr: u32, 343 // Msi Vary structure 344 msi_vary: MsiVary, 345 } 346 347 impl PciCapability for MsiCap { bytes(&self) -> &[u8]348 fn bytes(&self) -> &[u8] { 349 self.as_bytes() 350 } 351 id(&self) -> PciCapabilityID352 fn id(&self) -> PciCapabilityID { 353 PciCapabilityID::MessageSignalledInterrupts 354 } 355 writable_bits(&self) -> Vec<u32>356 fn writable_bits(&self) -> Vec<u32> { 357 // Check spec for detail 358 match ( 359 self.msg_ctl.get_is_64bit(), 360 self.msg_ctl.get_per_vector_masking(), 361 ) { 362 (0, 0) => vec![0x0471_0000, 0xffff_fffc, 0xffff_ffff], 363 (0, 1) => vec![0x0471_0000, 0xffff_fffc, 0xffff_ffff, 0xffff_ffff], 364 (1, 0) => vec![ 365 0x0471_0000, 366 0xffff_fffc, 367 0xffff_ffff, 368 0xffff_ffff, 369 0x0000_0000, 370 ], 371 (1, 1) => vec![ 372 0x0471_0000, 373 0xffff_fffc, 374 0xffff_ffff, 375 0xffff_ffff, 376 0xffff_ffff, 377 0x0000_0000, 378 ], 379 (_, _) => Vec::new(), 380 } 381 } 382 } 383 384 impl MsiCap { new(is_64bit: bool, mask_cap: bool) -> Self385 pub fn new(is_64bit: bool, mask_cap: bool) -> Self { 386 let mut msg_ctl = MsiCtrl::new(); 387 if is_64bit { 388 msg_ctl.set_is_64bit(1); 389 } 390 if mask_cap { 391 msg_ctl.set_per_vector_masking(1); 392 } 393 let msi_vary = match (is_64bit, mask_cap) { 394 (false, false) => MsiVary { 395 msi_32bit_without_mask: Msi32BitWithoutMask::default(), 396 }, 397 (false, true) => MsiVary { 398 msi_32bit_with_mask: Msi32BitWithMask::default(), 399 }, 400 (true, false) => MsiVary { 401 msi_64bit_without_mask: Msi64BitWithoutMask::default(), 402 }, 403 (true, true) => MsiVary { 404 msi_64bit_with_mask: Msi64BitWithMask::default(), 405 }, 406 }; 407 MsiCap { 408 _cap_vndr: 0, 409 _cap_next: 0, 410 msg_ctl, 411 msg_addr: 0, 412 msi_vary, 413 } 414 } 415 } 416 417 const MSI_CONFIG_READ_MASK: [u32; MSI_LENGTH_64BIT_WITH_MASK as usize / 4] = 418 [0xffff_0000, 0, 0, 0, 0, 0]; 419 420 impl PciCapConfig for MsiConfig { read_mask(&self) -> &'static [u32]421 fn read_mask(&self) -> &'static [u32] { 422 let num_regs = (self.len() + 3) / 4; 423 &MSI_CONFIG_READ_MASK[0..(num_regs as usize)] 424 } 425 read_reg(&self, reg_idx: usize) -> u32426 fn read_reg(&self, reg_idx: usize) -> u32 { 427 self.read_msi_capability(reg_idx as u32 * 4, 0) 428 } 429 write_reg( &mut self, reg_idx: usize, offset: u64, data: &[u8], ) -> Option<Box<dyn PciCapConfigWriteResult>>430 fn write_reg( 431 &mut self, 432 reg_idx: usize, 433 offset: u64, 434 data: &[u8], 435 ) -> Option<Box<dyn PciCapConfigWriteResult>> { 436 let offset = reg_idx as u32 * 4 + offset as u32; 437 self.write_msi_capability(offset, data); 438 None 439 } 440 } 441