// Copyright 2022 The ChromiumOS Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #![cfg_attr(windows, allow(dead_code))] use base::error; use base::AsRawDescriptor; use base::Event; use base::RawDescriptor; use base::Tube; use bit_field::*; use vm_control::VmIrqRequest; use vm_control::VmIrqResponse; use zerocopy::AsBytes; use zerocopy::FromBytes; use zerocopy::FromZeroes; use crate::pci::pci_configuration::PciCapConfig; use crate::pci::pci_configuration::PciCapConfigWriteResult; use crate::pci::PciCapability; use crate::pci::PciCapabilityID; // MSI registers pub const PCI_MSI_NEXT_POINTER: u32 = 0x1; // Next cap pointer pub const PCI_MSI_FLAGS: u32 = 0x2; // Message Control const PCI_MSI_FLAGS_ENABLE: u16 = 0x0001; // MSI feature enabled pub const PCI_MSI_FLAGS_64BIT: u16 = 0x0080; // 64-bit addresses allowed pub const PCI_MSI_FLAGS_MASKBIT: u16 = 0x0100; // Per-vector masking capable const PCI_MSI_ADDRESS_LO: u32 = 0x4; // MSI address lower 32 bits const PCI_MSI_ADDRESS_HI: u32 = 0x8; // MSI address upper 32 bits (if 64 bit allowed) const PCI_MSI_DATA_32: u32 = 0x8; // 16 bits of data for 32-bit message address const PCI_MSI_DATA_64: u32 = 0xC; // 16 bits of date for 64-bit message address // MSI length const MSI_LENGTH_32BIT_WITHOUT_MASK: u32 = 0xA; const MSI_LENGTH_32BIT_WITH_MASK: u32 = 0x14; const MSI_LENGTH_64BIT_WITHOUT_MASK: u32 = 0xE; const MSI_LENGTH_64BIT_WITH_MASK: u32 = 0x18; pub enum MsiStatus { Enabled, Disabled, NothingToDo, } /// Wrapper over MSI Capability Structure pub struct MsiConfig { is_64bit: bool, mask_cap: bool, ctrl: u16, address: u64, data: u16, vm_socket_irq: Tube, irqfd: Option, gsi: Option, device_id: u32, device_name: String, } impl MsiConfig { pub fn new( is_64bit: bool, mask_cap: bool, vm_socket_irq: Tube, device_id: u32, device_name: String, ) -> Self { let mut ctrl: u16 = 0; if is_64bit { ctrl |= PCI_MSI_FLAGS_64BIT; } if mask_cap { ctrl |= PCI_MSI_FLAGS_MASKBIT; } MsiConfig { is_64bit, mask_cap, ctrl, address: 0, data: 0, vm_socket_irq, irqfd: None, gsi: None, device_id, device_name, } } fn len(&self) -> u32 { match (self.is_64bit, self.mask_cap) { (true, true) => MSI_LENGTH_64BIT_WITH_MASK, (true, false) => MSI_LENGTH_64BIT_WITHOUT_MASK, (false, true) => MSI_LENGTH_32BIT_WITH_MASK, (false, false) => MSI_LENGTH_32BIT_WITHOUT_MASK, } } pub fn is_msi_reg(&self, offset: u32, index: u64, len: usize) -> bool { let msi_len = self.len(); index >= offset as u64 && index + len as u64 <= (offset + msi_len) as u64 && len as u32 <= msi_len } pub fn read_msi_capability(&self, offset: u32, data: u32) -> u32 { if offset == 0 { (self.ctrl as u32) << 16 | (data & u16::MAX as u32) } else { data } } pub fn write_msi_capability(&mut self, offset: u32, data: &[u8]) -> MsiStatus { let len = data.len(); let mut ret = MsiStatus::NothingToDo; let old_address = self.address; let old_data = self.data; // write msi ctl if len == 2 && offset == PCI_MSI_FLAGS { let was_enabled = self.is_msi_enabled(); let value: [u8; 2] = [data[0], data[1]]; self.ctrl = u16::from_le_bytes(value); let is_enabled = self.is_msi_enabled(); if !was_enabled && is_enabled { self.enable(); ret = MsiStatus::Enabled; } else if was_enabled && !is_enabled { ret = MsiStatus::Disabled; } } else if len == 4 && offset == PCI_MSI_ADDRESS_LO && !self.is_64bit { //write 32 bit message address let value: [u8; 8] = [data[0], data[1], data[2], data[3], 0, 0, 0, 0]; self.address = u64::from_le_bytes(value); } else if len == 4 && offset == PCI_MSI_ADDRESS_LO && self.is_64bit { // write 64 bit message address low part let value: [u8; 8] = [data[0], data[1], data[2], data[3], 0, 0, 0, 0]; self.address &= !0xffffffff; self.address |= u64::from_le_bytes(value); } else if len == 4 && offset == PCI_MSI_ADDRESS_HI && self.is_64bit { //write 64 bit message address high part let value: [u8; 8] = [0, 0, 0, 0, data[0], data[1], data[2], data[3]]; self.address &= 0xffffffff; self.address |= u64::from_le_bytes(value); } else if len == 8 && offset == PCI_MSI_ADDRESS_LO && self.is_64bit { // write 64 bit message address let value: [u8; 8] = [ data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], ]; self.address = u64::from_le_bytes(value); } else if len == 2 && ((offset == PCI_MSI_DATA_32 && !self.is_64bit) || (offset == PCI_MSI_DATA_64 && self.is_64bit)) { // write message data let value: [u8; 2] = [data[0], data[1]]; self.data = u16::from_le_bytes(value); } if self.is_msi_enabled() && (old_address != self.address || old_data != self.data) { self.add_msi_route(); } ret } pub fn is_msi_enabled(&self) -> bool { self.ctrl & PCI_MSI_FLAGS_ENABLE == PCI_MSI_FLAGS_ENABLE } fn add_msi_route(&self) { let gsi = match self.gsi { Some(g) => g, None => { error!("Add msi route but gsi is none"); return; } }; if let Err(e) = self.vm_socket_irq.send(&VmIrqRequest::AddMsiRoute { gsi, msi_address: self.address, msi_data: self.data.into(), }) { error!("failed to send AddMsiRoute request at {:?}", e); return; } match self.vm_socket_irq.recv() { Ok(VmIrqResponse::Err(e)) => error!("failed to call AddMsiRoute request {:?}", e), Ok(_) => {} Err(e) => error!("failed to receive AddMsiRoute response {:?}", e), } } fn allocate_one_msi(&mut self) { let irqfd = match self.irqfd.take() { Some(e) => e, None => match Event::new() { Ok(e) => e, Err(e) => { error!("failed to create event: {:?}", e); return; } }, }; let request = VmIrqRequest::AllocateOneMsi { irqfd, device_id: self.device_id, queue_id: 0, device_name: self.device_name.clone(), }; let request_result = self.vm_socket_irq.send(&request); // Stash the irqfd in self immediately because we used take above. self.irqfd = match request { VmIrqRequest::AllocateOneMsi { irqfd, .. } => Some(irqfd), _ => unreachable!(), }; if let Err(e) = request_result { error!("failed to send AllocateOneMsi request: {:?}", e); return; } match self.vm_socket_irq.recv() { Ok(VmIrqResponse::AllocateOneMsi { gsi }) => self.gsi = Some(gsi), _ => error!("failed to receive AllocateOneMsi Response"), } } fn enable(&mut self) { if self.gsi.is_none() || self.irqfd.is_none() { self.allocate_one_msi(); } self.add_msi_route(); } pub fn get_irqfd(&self) -> Option<&Event> { self.irqfd.as_ref() } pub fn destroy(&mut self) { if let Some(gsi) = self.gsi { if let Some(irqfd) = self.irqfd.take() { let request = VmIrqRequest::ReleaseOneIrq { gsi, irqfd }; if self.vm_socket_irq.send(&request).is_ok() { let _ = self.vm_socket_irq.recv::(); } } } } /// Return the raw descriptor of the MSI device socket pub fn get_msi_socket(&self) -> RawDescriptor { self.vm_socket_irq.as_raw_descriptor() } pub fn trigger(&self) { if let Some(irqfd) = self.irqfd.as_ref() { irqfd.signal().unwrap(); } } } #[bitfield] #[derive(Copy, Clone, AsBytes, FromZeroes, FromBytes)] pub struct MsiCtrl { enable: B1, multi_msg_capable: B3, multi_msg_enable: B3, is_64bit: B1, per_vector_masking: B1, extended_msg_data_capable: B1, extended_msg_data_enable: B1, reserved: B5, } #[allow(dead_code)] #[repr(C, align(4))] #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] struct Msi32BitWithoutMask { msg_data: u16, msg_extended_data: u16, _padding: [u8; 12], } #[allow(dead_code)] #[repr(C, align(4))] #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] struct Msi32BitWithMask { msg_data: u16, msg_extended_data: u16, mask_bits: u32, pending_bits: u32, _padding: [u8; 4], } #[allow(dead_code)] #[repr(C, align(4))] #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] struct Msi64BitWithoutMask { msg_upper: u32, msg_data: u16, msg_extended_data: u16, _padding: [u8; 8], } #[allow(dead_code)] #[repr(C)] #[derive(Clone, Copy, Default, AsBytes, FromZeroes, FromBytes)] struct Msi64BitWithMask { msg_upper: u32, msg_data: u16, msg_extended_data: u16, mask_bits: u32, pending_bits: u32, } #[allow(dead_code)] #[repr(C)] #[derive(Clone, Copy, AsBytes, FromZeroes, FromBytes)] union MsiVary { msi_32bit_without_mask: Msi32BitWithoutMask, msi_32bit_with_mask: Msi32BitWithMask, msi_64bit_without_mask: Msi64BitWithoutMask, msi_64bit_with_mask: Msi64BitWithMask, } #[allow(dead_code)] #[repr(C)] #[derive(Clone, Copy, AsBytes, FromZeroes, FromBytes)] /// MSI Capability Structure pub struct MsiCap { // To make add_capability() happy _cap_vndr: u8, _cap_next: u8, // Message Control Register msg_ctl: MsiCtrl, // Message Address msg_addr: u32, // Msi Vary structure msi_vary: MsiVary, } impl PciCapability for MsiCap { fn bytes(&self) -> &[u8] { self.as_bytes() } fn id(&self) -> PciCapabilityID { PciCapabilityID::MessageSignalledInterrupts } fn writable_bits(&self) -> Vec { // Check spec for detail match ( self.msg_ctl.get_is_64bit(), self.msg_ctl.get_per_vector_masking(), ) { (0, 0) => vec![0x0471_0000, 0xffff_fffc, 0xffff_ffff], (0, 1) => vec![0x0471_0000, 0xffff_fffc, 0xffff_ffff, 0xffff_ffff], (1, 0) => vec![ 0x0471_0000, 0xffff_fffc, 0xffff_ffff, 0xffff_ffff, 0x0000_0000, ], (1, 1) => vec![ 0x0471_0000, 0xffff_fffc, 0xffff_ffff, 0xffff_ffff, 0xffff_ffff, 0x0000_0000, ], (_, _) => Vec::new(), } } } impl MsiCap { pub fn new(is_64bit: bool, mask_cap: bool) -> Self { let mut msg_ctl = MsiCtrl::new(); if is_64bit { msg_ctl.set_is_64bit(1); } if mask_cap { msg_ctl.set_per_vector_masking(1); } let msi_vary = match (is_64bit, mask_cap) { (false, false) => MsiVary { msi_32bit_without_mask: Msi32BitWithoutMask::default(), }, (false, true) => MsiVary { msi_32bit_with_mask: Msi32BitWithMask::default(), }, (true, false) => MsiVary { msi_64bit_without_mask: Msi64BitWithoutMask::default(), }, (true, true) => MsiVary { msi_64bit_with_mask: Msi64BitWithMask::default(), }, }; MsiCap { _cap_vndr: 0, _cap_next: 0, msg_ctl, msg_addr: 0, msi_vary, } } } const MSI_CONFIG_READ_MASK: [u32; MSI_LENGTH_64BIT_WITH_MASK as usize / 4] = [0xffff_0000, 0, 0, 0, 0, 0]; impl PciCapConfig for MsiConfig { fn read_mask(&self) -> &'static [u32] { let num_regs = (self.len() + 3) / 4; &MSI_CONFIG_READ_MASK[0..(num_regs as usize)] } fn read_reg(&self, reg_idx: usize) -> u32 { self.read_msi_capability(reg_idx as u32 * 4, 0) } fn write_reg( &mut self, reg_idx: usize, offset: u64, data: &[u8], ) -> Option> { let offset = reg_idx as u32 * 4 + offset as u32; self.write_msi_capability(offset, data); None } }