1/*************************************************************************************** 2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* 6* XiangShan is licensed under Mulan PSL v2. 7* You can use this software according to the terms and conditions of the Mulan PSL v2. 8* You may obtain a copy of Mulan PSL v2 at: 9* http://license.coscl.org.cn/MulanPSL2 10* 11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 14* 15* See the Mulan PSL v2 for more details. 16* 17* 18* Acknowledgement 19* 20* This implementation is inspired by several key papers: 21* [1] Glenn Reinman, Brad Calder, and Todd Austin. "[Fetch directed instruction prefetching.] 22* (https://doi.org/10.1109/MICRO.1999.809439)" 32nd Annual ACM/IEEE International Symposium on Microarchitecture 23* (MICRO). 1999. 24***************************************************************************************/ 25 26package xiangshan.frontend.icache 27 28import chisel3._ 29import chisel3.util._ 30import freechips.rocketchip.diplomacy.AddressSet 31import freechips.rocketchip.diplomacy.IdRange 32import freechips.rocketchip.diplomacy.LazyModule 33import freechips.rocketchip.diplomacy.LazyModuleImp 34import freechips.rocketchip.tilelink._ 35import freechips.rocketchip.util.BundleFieldBase 36import huancun.AliasField 37import huancun.PrefetchField 38import org.chipsalliance.cde.config.Parameters 39import utility._ 40import utils._ 41import xiangshan._ 42import xiangshan.cache._ 43import xiangshan.cache.mmu.TlbRequestIO 44import xiangshan.frontend._ 45 46case class ICacheParameters( 47 nSets: Int = 256, 48 nWays: Int = 4, 49 rowBits: Int = 64, 50 nTLBEntries: Int = 32, 51 tagECC: Option[String] = None, 52 dataECC: Option[String] = None, 53 replacer: Option[String] = Some("random"), 54 PortNumber: Int = 2, 55 nFetchMshr: Int = 4, 56 nPrefetchMshr: Int = 10, 57 nWayLookupSize: Int = 32, 58 DataCodeUnit: Int = 64, 59 ICacheDataBanks: Int = 8, 60 ICacheDataSRAMWidth: Int = 66, 61 // TODO: hard code, need delete 62 partWayNum: Int = 4, 63 nMMIOs: Int = 1, 64 blockBytes: Int = 64, 65 cacheCtrlAddressOpt: Option[AddressSet] = None 66) extends L1CacheParameters { 67 68 val setBytes: Int = nSets * blockBytes 69 val aliasBitsOpt: Option[Int] = Option.when(setBytes > pageSize)(log2Ceil(setBytes / pageSize)) 70 val reqFields: Seq[BundleFieldBase] = Seq( 71 PrefetchField(), 72 ReqSourceField() 73 ) ++ aliasBitsOpt.map(AliasField) 74 val echoFields: Seq[BundleFieldBase] = Nil 75 def tagCode: Code = Code.fromString(tagECC) 76 def dataCode: Code = Code.fromString(dataECC) 77 def replacement = ReplacementPolicy.fromString(replacer, nWays, nSets) 78} 79 80trait HasICacheParameters extends HasL1CacheParameters with HasInstrMMIOConst with HasIFUConst { 81 val cacheParams: ICacheParameters = icacheParameters 82 83 def ctrlUnitParamsOpt: Option[L1ICacheCtrlParams] = OptionWrapper( 84 cacheParams.cacheCtrlAddressOpt.nonEmpty, 85 L1ICacheCtrlParams( 86 address = cacheParams.cacheCtrlAddressOpt.get, 87 regWidth = XLEN 88 ) 89 ) 90 91 def ICacheSets: Int = cacheParams.nSets 92 def ICacheWays: Int = cacheParams.nWays 93 def PortNumber: Int = cacheParams.PortNumber 94 def nFetchMshr: Int = cacheParams.nFetchMshr 95 def nPrefetchMshr: Int = cacheParams.nPrefetchMshr 96 def nWayLookupSize: Int = cacheParams.nWayLookupSize 97 def DataCodeUnit: Int = cacheParams.DataCodeUnit 98 def ICacheDataBanks: Int = cacheParams.ICacheDataBanks 99 def ICacheDataSRAMWidth: Int = cacheParams.ICacheDataSRAMWidth 100 def partWayNum: Int = cacheParams.partWayNum 101 102 def ICacheMetaBits: Int = tagBits // FIXME: unportable: maybe use somemethod to get width 103 def ICacheMetaCodeBits: Int = 1 // FIXME: unportable: maybe use cacheParams.tagCode.somemethod to get width 104 def ICacheMetaEntryBits: Int = ICacheMetaBits + ICacheMetaCodeBits 105 106 def ICacheDataBits: Int = blockBits / ICacheDataBanks 107 def ICacheDataCodeSegs: Int = 108 math.ceil(ICacheDataBits / DataCodeUnit).toInt // split data to segments for ECC checking 109 def ICacheDataCodeBits: Int = 110 ICacheDataCodeSegs * 1 // FIXME: unportable: maybe use cacheParams.dataCode.somemethod to get width 111 def ICacheDataEntryBits: Int = ICacheDataBits + ICacheDataCodeBits 112 def ICacheBankVisitNum: Int = 32 * 8 / ICacheDataBits + 1 113 def highestIdxBit: Int = log2Ceil(nSets) - 1 114 115 require((ICacheDataBanks >= 2) && isPow2(ICacheDataBanks)) 116 require(ICacheDataSRAMWidth >= ICacheDataEntryBits) 117 require(isPow2(ICacheSets), s"nSets($ICacheSets) must be pow2") 118 require(isPow2(ICacheWays), s"nWays($ICacheWays) must be pow2") 119 120 def generatePipeControl(lastFire: Bool, thisFire: Bool, thisFlush: Bool, lastFlush: Bool): Bool = { 121 val valid = RegInit(false.B) 122 when(thisFlush)(valid := false.B) 123 .elsewhen(lastFire && !lastFlush)(valid := true.B) 124 .elsewhen(thisFire)(valid := false.B) 125 valid 126 } 127 128 def ResultHoldBypass[T <: Data](data: T, valid: Bool): T = 129 Mux(valid, data, RegEnable(data, valid)) 130 131 def ResultHoldBypass[T <: Data](data: T, init: T, valid: Bool): T = 132 Mux(valid, data, RegEnable(data, init, valid)) 133 134 def holdReleaseLatch(valid: Bool, release: Bool, flush: Bool): Bool = { 135 val bit = RegInit(false.B) 136 when(flush)(bit := false.B) 137 .elsewhen(valid && !release)(bit := true.B) 138 .elsewhen(release)(bit := false.B) 139 bit || valid 140 } 141 142 def blockCounter(block: Bool, flush: Bool, threshold: Int): Bool = { 143 val counter = RegInit(0.U(log2Up(threshold + 1).W)) 144 when(block)(counter := counter + 1.U) 145 when(flush)(counter := 0.U) 146 counter > threshold.U 147 } 148 149 def InitQueue[T <: Data](entry: T, size: Int): Vec[T] = 150 RegInit(VecInit(Seq.fill(size)(0.U.asTypeOf(entry.cloneType)))) 151 152 def getBankSel(blkOffset: UInt, valid: Bool = true.B): Vec[UInt] = { 153 val bankIdxLow = (Cat(0.U(1.W), blkOffset) >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt 154 val bankIdxHigh = ((Cat(0.U(1.W), blkOffset) + 32.U) >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt 155 val bankSel = VecInit((0 until ICacheDataBanks * 2).map(i => (i.U >= bankIdxLow) && (i.U <= bankIdxHigh))) 156 assert( 157 !valid || PopCount(bankSel) === ICacheBankVisitNum.U, 158 "The number of bank visits must be %d, but bankSel=0x%x", 159 ICacheBankVisitNum.U, 160 bankSel.asUInt 161 ) 162 bankSel.asTypeOf(UInt((ICacheDataBanks * 2).W)).asTypeOf(Vec(2, UInt(ICacheDataBanks.W))) 163 } 164 165 def getLineSel(blkOffset: UInt): Vec[Bool] = { 166 val bankIdxLow = (blkOffset >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt 167 val lineSel = VecInit((0 until ICacheDataBanks).map(i => i.U < bankIdxLow)) 168 lineSel 169 } 170 171 def getBlkAddr(addr: UInt): UInt = (addr >> blockOffBits).asUInt 172 def getPhyTagFromBlk(addr: UInt): UInt = (addr >> (pgUntagBits - blockOffBits)).asUInt 173 def getIdxFromBlk(addr: UInt): UInt = addr(idxBits - 1, 0) 174 def getPaddrFromPtag(vaddr: UInt, ptag: UInt): UInt = Cat(ptag, vaddr(pgUntagBits - 1, 0)) 175 def getPaddrFromPtag(vaddrVec: Vec[UInt], ptagVec: Vec[UInt]): Vec[UInt] = 176 VecInit((vaddrVec zip ptagVec).map { case (vaddr, ptag) => getPaddrFromPtag(vaddr, ptag) }) 177} 178 179trait HasICacheECCHelper extends HasICacheParameters { 180 def encodeMetaECC(meta: UInt, poison: Bool = false.B): UInt = { 181 require(meta.getWidth == ICacheMetaBits) 182 val code = cacheParams.tagCode.encode(meta, poison) >> ICacheMetaBits 183 code.asTypeOf(UInt(ICacheMetaCodeBits.W)) 184 } 185 186 def encodeDataECC(data: UInt, poison: Bool = false.B): UInt = { 187 require(data.getWidth == ICacheDataBits) 188 val datas = data.asTypeOf(Vec(ICacheDataCodeSegs, UInt((ICacheDataBits / ICacheDataCodeSegs).W))) 189 val codes = VecInit(datas.map(cacheParams.dataCode.encode(_, poison) >> (ICacheDataBits / ICacheDataCodeSegs))) 190 codes.asTypeOf(UInt(ICacheDataCodeBits.W)) 191 } 192} 193 194abstract class ICacheBundle(implicit p: Parameters) extends XSBundle 195 with HasICacheParameters 196 197abstract class ICacheModule(implicit p: Parameters) extends XSModule 198 with HasICacheParameters 199 200abstract class ICacheArray(implicit p: Parameters) extends XSModule 201 with HasICacheParameters 202 203class ICacheMetadata(implicit p: Parameters) extends ICacheBundle { 204 val tag: UInt = UInt(tagBits.W) 205} 206 207object ICacheMetadata { 208 def apply(tag: Bits)(implicit p: Parameters): ICacheMetadata = { 209 val meta = Wire(new ICacheMetadata) 210 meta.tag := tag 211 meta 212 } 213} 214 215class ICacheMetaArrayIO(implicit p: Parameters) extends ICacheBundle { 216 val write: DecoupledIO[ICacheMetaWriteBundle] = Flipped(DecoupledIO(new ICacheMetaWriteBundle)) 217 val read: DecoupledIO[ICacheReadBundle] = Flipped(DecoupledIO(new ICacheReadBundle)) 218 val readResp: ICacheMetaRespBundle = Output(new ICacheMetaRespBundle) 219 val flush: Vec[Valid[ICacheMetaFlushBundle]] = Vec(PortNumber, Flipped(ValidIO(new ICacheMetaFlushBundle))) 220 val flushAll: Bool = Input(Bool()) 221} 222 223class ICacheMetaArray(implicit p: Parameters) extends ICacheArray with HasICacheECCHelper { 224 class ICacheMetaEntry(implicit p: Parameters) extends ICacheBundle { 225 val meta: ICacheMetadata = new ICacheMetadata 226 val code: UInt = UInt(ICacheMetaCodeBits.W) 227 } 228 229 private object ICacheMetaEntry { 230 def apply(meta: ICacheMetadata, poison: Bool)(implicit p: Parameters): ICacheMetaEntry = { 231 val entry = Wire(new ICacheMetaEntry) 232 entry.meta := meta 233 entry.code := encodeMetaECC(meta.asUInt, poison) 234 entry 235 } 236 } 237 238 // sanity check 239 require(ICacheMetaEntryBits == (new ICacheMetaEntry).getWidth) 240 241 val io: ICacheMetaArrayIO = IO(new ICacheMetaArrayIO) 242 243 private val port_0_read_0 = io.read.valid && !io.read.bits.vSetIdx(0)(0) 244 private val port_0_read_1 = io.read.valid && io.read.bits.vSetIdx(0)(0) 245 private val port_1_read_1 = io.read.valid && io.read.bits.vSetIdx(1)(0) && io.read.bits.isDoubleLine 246 private val port_1_read_0 = io.read.valid && !io.read.bits.vSetIdx(1)(0) && io.read.bits.isDoubleLine 247 248 private val port_0_read_0_reg = RegEnable(port_0_read_0, 0.U.asTypeOf(port_0_read_0), io.read.fire) 249 private val port_0_read_1_reg = RegEnable(port_0_read_1, 0.U.asTypeOf(port_0_read_1), io.read.fire) 250 private val port_1_read_1_reg = RegEnable(port_1_read_1, 0.U.asTypeOf(port_1_read_1), io.read.fire) 251 private val port_1_read_0_reg = RegEnable(port_1_read_0, 0.U.asTypeOf(port_1_read_0), io.read.fire) 252 253 private val bank_0_idx = Mux(port_0_read_0, io.read.bits.vSetIdx(0), io.read.bits.vSetIdx(1)) 254 private val bank_1_idx = Mux(port_0_read_1, io.read.bits.vSetIdx(0), io.read.bits.vSetIdx(1)) 255 256 private val write_bank_0 = io.write.valid && !io.write.bits.bankIdx 257 private val write_bank_1 = io.write.valid && io.write.bits.bankIdx 258 259 private val write_meta_bits = ICacheMetaEntry( 260 meta = ICacheMetadata( 261 tag = io.write.bits.phyTag 262 ), 263 poison = io.write.bits.poison 264 ) 265 266 private val tagArrays = (0 until PortNumber) map { bank => 267 val tagArray = Module(new SRAMTemplate( 268 new ICacheMetaEntry(), 269 set = nSets / PortNumber, 270 way = nWays, 271 shouldReset = true, 272 holdRead = true, 273 singlePort = true, 274 withClockGate = false // enable signal timing is bad, no gating here 275 )) 276 277 // meta connection 278 if (bank == 0) { 279 tagArray.io.r.req.valid := port_0_read_0 || port_1_read_0 280 tagArray.io.r.req.bits.apply(setIdx = bank_0_idx(highestIdxBit, 1)) 281 tagArray.io.w.req.valid := write_bank_0 282 tagArray.io.w.req.bits.apply( 283 data = write_meta_bits, 284 setIdx = io.write.bits.virIdx(highestIdxBit, 1), 285 waymask = io.write.bits.waymask 286 ) 287 } else { 288 tagArray.io.r.req.valid := port_0_read_1 || port_1_read_1 289 tagArray.io.r.req.bits.apply(setIdx = bank_1_idx(highestIdxBit, 1)) 290 tagArray.io.w.req.valid := write_bank_1 291 tagArray.io.w.req.bits.apply( 292 data = write_meta_bits, 293 setIdx = io.write.bits.virIdx(highestIdxBit, 1), 294 waymask = io.write.bits.waymask 295 ) 296 } 297 298 tagArray 299 } 300 301 private val read_set_idx_next = RegEnable(io.read.bits.vSetIdx, 0.U.asTypeOf(io.read.bits.vSetIdx), io.read.fire) 302 private val valid_array = RegInit(VecInit(Seq.fill(nWays)(0.U(nSets.W)))) 303 private val valid_metas = Wire(Vec(PortNumber, Vec(nWays, Bool()))) 304 // valid read 305 (0 until PortNumber).foreach(i => 306 (0 until nWays).foreach(way => 307 valid_metas(i)(way) := valid_array(way)(read_set_idx_next(i)) 308 ) 309 ) 310 io.readResp.entryValid := valid_metas 311 312 io.read.ready := !io.write.valid && !io.flush.map(_.valid).reduce(_ || _) && !io.flushAll && 313 tagArrays.map(_.io.r.req.ready).reduce(_ && _) 314 315 // valid write 316 private val way_num = OHToUInt(io.write.bits.waymask) 317 when(io.write.valid) { 318 valid_array(way_num) := valid_array(way_num).bitSet(io.write.bits.virIdx, true.B) 319 } 320 321 XSPerfAccumulate("meta_refill_num", io.write.valid) 322 323 io.readResp.metas <> DontCare 324 io.readResp.codes <> DontCare 325 private val readMetaEntries = tagArrays.map(port => port.io.r.resp.asTypeOf(Vec(nWays, new ICacheMetaEntry()))) 326 private val readMetas = readMetaEntries.map(_.map(_.meta)) 327 private val readCodes = readMetaEntries.map(_.map(_.code)) 328 329 // TEST: force ECC to fail by setting readCodes to 0 330 if (ICacheForceMetaECCError) { 331 readCodes.foreach(_.foreach(_ := 0.U)) 332 } 333 334 when(port_0_read_0_reg) { 335 io.readResp.metas(0) := readMetas(0) 336 io.readResp.codes(0) := readCodes(0) 337 }.elsewhen(port_0_read_1_reg) { 338 io.readResp.metas(0) := readMetas(1) 339 io.readResp.codes(0) := readCodes(1) 340 } 341 342 when(port_1_read_0_reg) { 343 io.readResp.metas(1) := readMetas(0) 344 io.readResp.codes(1) := readCodes(0) 345 }.elsewhen(port_1_read_1_reg) { 346 io.readResp.metas(1) := readMetas(1) 347 io.readResp.codes(1) := readCodes(1) 348 } 349 350 io.write.ready := true.B // TODO : has bug ? should be !io.cacheOp.req.valid 351 352 /* 353 * flush logic 354 */ 355 // flush standalone set (e.g. flushed by mainPipe before doing re-fetch) 356 when(io.flush.map(_.valid).reduce(_ || _)) { 357 (0 until nWays).foreach { w => 358 valid_array(w) := (0 until PortNumber).map { i => 359 Mux( 360 // check if set `virIdx` in way `w` is requested to be flushed by port `i` 361 io.flush(i).valid && io.flush(i).bits.waymask(w), 362 valid_array(w).bitSet(io.flush(i).bits.virIdx, false.B), 363 valid_array(w) 364 ) 365 }.reduce(_ & _) 366 } 367 } 368 369 // flush all (e.g. fence.i) 370 when(io.flushAll) { 371 (0 until nWays).foreach(w => valid_array(w) := 0.U) 372 } 373 374 // PERF: flush counter 375 XSPerfAccumulate("flush", io.flush.map(_.valid).reduce(_ || _)) 376 XSPerfAccumulate("flush_all", io.flushAll) 377} 378 379class ICacheDataArrayIO(implicit p: Parameters) extends ICacheBundle { 380 val write: DecoupledIO[ICacheDataWriteBundle] = Flipped(DecoupledIO(new ICacheDataWriteBundle)) 381 val read: Vec[DecoupledIO[ICacheReadBundle]] = Flipped(Vec(partWayNum, DecoupledIO(new ICacheReadBundle))) 382 val readResp: ICacheDataRespBundle = Output(new ICacheDataRespBundle) 383} 384 385class ICacheDataArray(implicit p: Parameters) extends ICacheArray with HasICacheECCHelper { 386 class ICacheDataEntry(implicit p: Parameters) extends ICacheBundle { 387 val data: UInt = UInt(ICacheDataBits.W) 388 val code: UInt = UInt(ICacheDataCodeBits.W) 389 } 390 391 private object ICacheDataEntry { 392 def apply(data: UInt, poison: Bool)(implicit p: Parameters): ICacheDataEntry = { 393 val entry = Wire(new ICacheDataEntry) 394 entry.data := data 395 entry.code := encodeDataECC(data, poison) 396 entry 397 } 398 } 399 400 val io: ICacheDataArrayIO = IO(new ICacheDataArrayIO) 401 402 /** 403 ****************************************************************************** 404 * data array 405 ****************************************************************************** 406 */ 407 private val writeDatas = io.write.bits.data.asTypeOf(Vec(ICacheDataBanks, UInt(ICacheDataBits.W))) 408 private val writeEntries = writeDatas.map(ICacheDataEntry(_, io.write.bits.poison).asUInt) 409 410 // io.read() are copies to control fan-out, we can simply use .head here 411 private val bankSel = getBankSel(io.read.head.bits.blkOffset, io.read.head.valid) 412 private val lineSel = getLineSel(io.read.head.bits.blkOffset) 413 private val waymasks = io.read.head.bits.waymask 414 private val masks = Wire(Vec(nWays, Vec(ICacheDataBanks, Bool()))) 415 (0 until nWays).foreach { way => 416 (0 until ICacheDataBanks).foreach { bank => 417 masks(way)(bank) := Mux( 418 lineSel(bank), 419 waymasks(1)(way) && bankSel(1)(bank).asBool, 420 waymasks(0)(way) && bankSel(0)(bank).asBool 421 ) 422 } 423 } 424 425 private val dataArrays = (0 until nWays).map { way => 426 (0 until ICacheDataBanks).map { bank => 427 val sramBank = Module(new SRAMTemplateWithFixedWidth( 428 UInt(ICacheDataEntryBits.W), 429 set = nSets, 430 width = ICacheDataSRAMWidth, 431 shouldReset = true, 432 holdRead = true, 433 singlePort = true, 434 withClockGate = true 435 )) 436 437 // read 438 sramBank.io.r.req.valid := io.read(bank % 4).valid && masks(way)(bank) 439 sramBank.io.r.req.bits.apply(setIdx = 440 Mux(lineSel(bank), io.read(bank % 4).bits.vSetIdx(1), io.read(bank % 4).bits.vSetIdx(0)) 441 ) 442 // write 443 sramBank.io.w.req.valid := io.write.valid && io.write.bits.waymask(way).asBool 444 sramBank.io.w.req.bits.apply( 445 data = writeEntries(bank), 446 setIdx = io.write.bits.virIdx, 447 // waymask is invalid when way of SRAMTemplate <= 1 448 waymask = 0.U 449 ) 450 sramBank 451 } 452 } 453 454 /** 455 ****************************************************************************** 456 * read logic 457 ****************************************************************************** 458 */ 459 private val masksReg = RegEnable(masks, 0.U.asTypeOf(masks), io.read(0).valid) 460 private val readDataWithCode = (0 until ICacheDataBanks).map { bank => 461 Mux1H(VecInit(masksReg.map(_(bank))).asTypeOf(UInt(nWays.W)), dataArrays.map(_(bank).io.r.resp.asUInt)) 462 } 463 private val readEntries = readDataWithCode.map(_.asTypeOf(new ICacheDataEntry())) 464 private val readDatas = VecInit(readEntries.map(_.data)) 465 private val readCodes = VecInit(readEntries.map(_.code)) 466 467 // TEST: force ECC to fail by setting readCodes to 0 468 if (ICacheForceDataECCError) { 469 readCodes.foreach(_ := 0.U) 470 } 471 472 /** 473 ****************************************************************************** 474 * IO 475 ****************************************************************************** 476 */ 477 io.readResp.datas := readDatas 478 io.readResp.codes := readCodes 479 io.write.ready := true.B 480 io.read.foreach(_.ready := !io.write.valid) 481} 482 483class ICacheReplacerIO(implicit p: Parameters) extends ICacheBundle { 484 val touch: Vec[Valid[ReplacerTouch]] = Vec(PortNumber, Flipped(ValidIO(new ReplacerTouch))) 485 val victim: ReplacerVictim = Flipped(new ReplacerVictim) 486} 487 488class ICacheReplacer(implicit p: Parameters) extends ICacheModule { 489 val io: ICacheReplacerIO = IO(new ICacheReplacerIO) 490 491 private val replacers = 492 Seq.fill(PortNumber)(ReplacementPolicy.fromString(cacheParams.replacer, nWays, nSets / PortNumber)) 493 494 // touch 495 private val touch_sets = Seq.fill(PortNumber)(Wire(Vec(PortNumber, UInt(log2Ceil(nSets / PortNumber).W)))) 496 private val touch_ways = Seq.fill(PortNumber)(Wire(Vec(PortNumber, Valid(UInt(wayBits.W))))) 497 (0 until PortNumber).foreach { i => 498 touch_sets(i)(0) := Mux( 499 io.touch(i).bits.vSetIdx(0), 500 io.touch(1).bits.vSetIdx(highestIdxBit, 1), 501 io.touch(0).bits.vSetIdx(highestIdxBit, 1) 502 ) 503 touch_ways(i)(0).bits := Mux(io.touch(i).bits.vSetIdx(0), io.touch(1).bits.way, io.touch(0).bits.way) 504 touch_ways(i)(0).valid := Mux(io.touch(i).bits.vSetIdx(0), io.touch(1).valid, io.touch(0).valid) 505 } 506 507 // victim 508 io.victim.way := Mux( 509 io.victim.vSetIdx.bits(0), 510 replacers(1).way(io.victim.vSetIdx.bits(highestIdxBit, 1)), 511 replacers(0).way(io.victim.vSetIdx.bits(highestIdxBit, 1)) 512 ) 513 514 // touch the victim in next cycle 515 private val victim_vSetIdx_reg = 516 RegEnable(io.victim.vSetIdx.bits, 0.U.asTypeOf(io.victim.vSetIdx.bits), io.victim.vSetIdx.valid) 517 private val victim_way_reg = RegEnable(io.victim.way, 0.U.asTypeOf(io.victim.way), io.victim.vSetIdx.valid) 518 (0 until PortNumber).foreach { i => 519 touch_sets(i)(1) := victim_vSetIdx_reg(highestIdxBit, 1) 520 touch_ways(i)(1).bits := victim_way_reg 521 touch_ways(i)(1).valid := RegNext(io.victim.vSetIdx.valid) && (victim_vSetIdx_reg(0) === i.U) 522 } 523 524 ((replacers zip touch_sets) zip touch_ways).foreach { case ((r, s), w) => r.access(s, w) } 525} 526 527class ICacheIO(implicit p: Parameters) extends ICacheBundle { 528 val hartId: UInt = Input(UInt(hartIdLen.W)) 529 // FTQ 530 val fetch: ICacheMainPipeBundle = new ICacheMainPipeBundle 531 val ftqPrefetch: FtqToPrefetchIO = Flipped(new FtqToPrefetchIO) 532 // memblock 533 val softPrefetch: Vec[Valid[SoftIfetchPrefetchBundle]] = 534 Vec(backendParams.LduCnt, Flipped(Valid(new SoftIfetchPrefetchBundle))) 535 // IFU 536 val stop: Bool = Input(Bool()) 537 val toIFU: Bool = Output(Bool()) 538 // PMP: mainPipe & prefetchPipe need PortNumber each 539 val pmp: Vec[ICachePMPBundle] = Vec(2 * PortNumber, new ICachePMPBundle) 540 // iTLB 541 val itlb: Vec[TlbRequestIO] = Vec(PortNumber, new TlbRequestIO) 542 val itlbFlushPipe: Bool = Bool() 543 // backend/BEU 544 val error: Valid[L1CacheErrorInfo] = ValidIO(new L1CacheErrorInfo) 545 // backend/CSR 546 val csr_pf_enable: Bool = Input(Bool()) 547 // flush 548 val fencei: Bool = Input(Bool()) 549 val flush: Bool = Input(Bool()) 550 551 // perf 552 val perfInfo: ICachePerfInfo = Output(new ICachePerfInfo) 553} 554 555class ICache()(implicit p: Parameters) extends LazyModule with HasICacheParameters { 556 override def shouldBeInlined: Boolean = false 557 558 val clientParameters: TLMasterPortParameters = TLMasterPortParameters.v1( 559 Seq(TLMasterParameters.v1( 560 name = "icache", 561 sourceId = IdRange(0, cacheParams.nFetchMshr + cacheParams.nPrefetchMshr + 1) 562 )), 563 requestFields = cacheParams.reqFields, 564 echoFields = cacheParams.echoFields 565 ) 566 567 val clientNode: TLClientNode = TLClientNode(Seq(clientParameters)) 568 569 val ctrlUnitOpt: Option[ICacheCtrlUnit] = ctrlUnitParamsOpt.map(params => LazyModule(new ICacheCtrlUnit(params))) 570 571 lazy val module: ICacheImp = new ICacheImp(this) 572} 573 574class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParameters with HasPerfEvents { 575 val io: ICacheIO = IO(new ICacheIO) 576 577 println("ICache:") 578 println(" TagECC: " + cacheParams.tagECC) 579 println(" DataECC: " + cacheParams.dataECC) 580 println(" ICacheSets: " + cacheParams.nSets) 581 println(" ICacheWays: " + cacheParams.nWays) 582 println(" PortNumber: " + cacheParams.PortNumber) 583 println(" nFetchMshr: " + cacheParams.nFetchMshr) 584 println(" nPrefetchMshr: " + cacheParams.nPrefetchMshr) 585 println(" nWayLookupSize: " + cacheParams.nWayLookupSize) 586 println(" DataCodeUnit: " + cacheParams.DataCodeUnit) 587 println(" ICacheDataBanks: " + cacheParams.ICacheDataBanks) 588 println(" ICacheDataSRAMWidth: " + cacheParams.ICacheDataSRAMWidth) 589 590 val (bus, edge) = outer.clientNode.out.head 591 592 private val metaArray = Module(new ICacheMetaArray) 593 private val dataArray = Module(new ICacheDataArray) 594 private val mainPipe = Module(new ICacheMainPipe) 595 private val missUnit = Module(new ICacheMissUnit(edge)) 596 private val replacer = Module(new ICacheReplacer) 597 private val prefetcher = Module(new IPrefetchPipe) 598 private val wayLookup = Module(new WayLookup) 599 600 private val ecc_enable = if (outer.ctrlUnitOpt.nonEmpty) outer.ctrlUnitOpt.get.module.io.ecc_enable else true.B 601 602 // dataArray io 603 if (outer.ctrlUnitOpt.nonEmpty) { 604 val ctrlUnit = outer.ctrlUnitOpt.get.module 605 when(ctrlUnit.io.injecting) { 606 dataArray.io.write <> ctrlUnit.io.dataWrite 607 missUnit.io.data_write.ready := false.B 608 }.otherwise { 609 ctrlUnit.io.dataWrite.ready := false.B 610 dataArray.io.write <> missUnit.io.data_write 611 } 612 } else { 613 dataArray.io.write <> missUnit.io.data_write 614 } 615 dataArray.io.read <> mainPipe.io.dataArray.toIData 616 mainPipe.io.dataArray.fromIData := dataArray.io.readResp 617 618 // metaArray io 619 metaArray.io.flushAll := io.fencei 620 metaArray.io.flush <> mainPipe.io.metaArrayFlush 621 if (outer.ctrlUnitOpt.nonEmpty) { 622 val ctrlUnit = outer.ctrlUnitOpt.get.module 623 when(ctrlUnit.io.injecting) { 624 metaArray.io.write <> ctrlUnit.io.metaWrite 625 metaArray.io.read <> ctrlUnit.io.metaRead 626 missUnit.io.meta_write.ready := false.B 627 prefetcher.io.metaRead.toIMeta.ready := false.B 628 }.otherwise { 629 ctrlUnit.io.metaWrite.ready := false.B 630 ctrlUnit.io.metaRead.ready := false.B 631 metaArray.io.write <> missUnit.io.meta_write 632 metaArray.io.read <> prefetcher.io.metaRead.toIMeta 633 } 634 ctrlUnit.io.metaReadResp := metaArray.io.readResp 635 } else { 636 metaArray.io.write <> missUnit.io.meta_write 637 metaArray.io.read <> prefetcher.io.metaRead.toIMeta 638 } 639 prefetcher.io.metaRead.fromIMeta := metaArray.io.readResp 640 641 prefetcher.io.flush := io.flush 642 prefetcher.io.csr_pf_enable := io.csr_pf_enable 643 prefetcher.io.ecc_enable := ecc_enable 644 prefetcher.io.MSHRResp := missUnit.io.fetch_resp 645 prefetcher.io.flushFromBpu := io.ftqPrefetch.flushFromBpu 646 // cache softPrefetch 647 private val softPrefetchValid = RegInit(false.B) 648 private val softPrefetch = RegInit(0.U.asTypeOf(new IPrefetchReq)) 649 /* FIXME: 650 * If there is already a pending softPrefetch request, it will be overwritten. 651 * Also, if there are multiple softPrefetch requests in the same cycle, only the first one will be accepted. 652 * We should implement a softPrefetchQueue (like ibuffer, multi-in, single-out) to solve this. 653 * However, the impact on performance still needs to be assessed. 654 * Considering that the frequency of prefetch.i may not be high, let's start with a temporary dummy solution. 655 */ 656 when(io.softPrefetch.map(_.valid).reduce(_ || _)) { 657 softPrefetchValid := true.B 658 softPrefetch.fromSoftPrefetch(MuxCase( 659 0.U.asTypeOf(new SoftIfetchPrefetchBundle), 660 io.softPrefetch.map(req => req.valid -> req.bits) 661 )) 662 }.elsewhen(prefetcher.io.req.fire) { 663 softPrefetchValid := false.B 664 } 665 // pass ftqPrefetch 666 private val ftqPrefetch = WireInit(0.U.asTypeOf(new IPrefetchReq)) 667 ftqPrefetch.fromFtqICacheInfo(io.ftqPrefetch.req.bits) 668 // software prefetch has higher priority 669 prefetcher.io.req.valid := softPrefetchValid || io.ftqPrefetch.req.valid 670 prefetcher.io.req.bits := Mux(softPrefetchValid, softPrefetch, ftqPrefetch) 671 prefetcher.io.req.bits.backendException := io.ftqPrefetch.backendException 672 io.ftqPrefetch.req.ready := prefetcher.io.req.ready && !softPrefetchValid 673 674 missUnit.io.hartId := io.hartId 675 missUnit.io.fencei := io.fencei 676 missUnit.io.flush := io.flush 677 missUnit.io.fetch_req <> mainPipe.io.mshr.req 678 missUnit.io.prefetch_req <> prefetcher.io.MSHRReq 679 missUnit.io.mem_grant.valid := false.B 680 missUnit.io.mem_grant.bits := DontCare 681 missUnit.io.mem_grant <> bus.d 682 683 mainPipe.io.flush := io.flush 684 mainPipe.io.respStall := io.stop 685 mainPipe.io.ecc_enable := ecc_enable 686 mainPipe.io.hartId := io.hartId 687 mainPipe.io.mshr.resp := missUnit.io.fetch_resp 688 mainPipe.io.fetch.req <> io.fetch.req 689 mainPipe.io.wayLookupRead <> wayLookup.io.read 690 691 wayLookup.io.flush := io.flush 692 wayLookup.io.write <> prefetcher.io.wayLookupWrite 693 wayLookup.io.update := missUnit.io.fetch_resp 694 695 replacer.io.touch <> mainPipe.io.touch 696 replacer.io.victim <> missUnit.io.victim 697 698 io.pmp(0) <> mainPipe.io.pmp(0) 699 io.pmp(1) <> mainPipe.io.pmp(1) 700 io.pmp(2) <> prefetcher.io.pmp(0) 701 io.pmp(3) <> prefetcher.io.pmp(1) 702 703 io.itlb(0) <> prefetcher.io.itlb(0) 704 io.itlb(1) <> prefetcher.io.itlb(1) 705 io.itlbFlushPipe := prefetcher.io.itlbFlushPipe 706 707 // notify IFU that Icache pipeline is available 708 io.toIFU := mainPipe.io.fetch.req.ready 709 io.perfInfo := mainPipe.io.perfInfo 710 711 io.fetch.resp <> mainPipe.io.fetch.resp 712 io.fetch.topdownIcacheMiss := mainPipe.io.fetch.topdownIcacheMiss 713 io.fetch.topdownItlbMiss := mainPipe.io.fetch.topdownItlbMiss 714 715 bus.b.ready := false.B 716 bus.c.valid := false.B 717 bus.c.bits := DontCare 718 bus.e.valid := false.B 719 bus.e.bits := DontCare 720 721 bus.a <> missUnit.io.mem_acquire 722 723 // Parity error port 724 private val errors = mainPipe.io.errors 725 private val errors_valid = errors.map(e => e.valid).reduce(_ | _) 726 io.error.bits <> RegEnable( 727 PriorityMux(errors.map(e => e.valid -> e.bits)), 728 0.U.asTypeOf(errors(0).bits), 729 errors_valid 730 ) 731 io.error.valid := RegNext(errors_valid, false.B) 732 733 XSPerfAccumulate( 734 "softPrefetch_drop_not_ready", 735 io.softPrefetch.map(_.valid).reduce(_ || _) && softPrefetchValid && !prefetcher.io.req.fire 736 ) 737 XSPerfAccumulate("softPrefetch_drop_multi_req", PopCount(io.softPrefetch.map(_.valid)) > 1.U) 738 XSPerfAccumulate("softPrefetch_block_ftq", softPrefetchValid && io.ftqPrefetch.req.valid) 739 740 val perfEvents: Seq[(String, Bool)] = Seq( 741 ("icache_miss_cnt ", false.B), 742 ("icache_miss_penalty", BoolStopWatch(start = false.B, stop = false.B || false.B, startHighPriority = true)) 743 ) 744 generatePerfEvent() 745} 746 747//class ICachePartWayReadBundle[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) 748// extends ICacheBundle { 749// val req = Flipped(Vec( 750// PortNumber, 751// Decoupled(new Bundle { 752// val ridx = UInt((log2Ceil(nSets) - 1).W) 753// }) 754// )) 755// val resp = Output(new Bundle { 756// val rdata = Vec(PortNumber, Vec(pWay, gen)) 757// }) 758//} 759 760//class ICacheWriteBundle[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) 761// extends ICacheBundle { 762// val wdata = gen 763// val widx = UInt((log2Ceil(nSets) - 1).W) 764// val wbankidx = Bool() 765// val wmask = Vec(pWay, Bool()) 766//} 767 768//class ICachePartWayArray[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) extends ICacheArray { 769// 770// // including part way data 771// val io = IO { 772// new Bundle { 773// val read = new ICachePartWayReadBundle(gen, pWay) 774// val write = Flipped(ValidIO(new ICacheWriteBundle(gen, pWay))) 775// } 776// } 777// 778// io.read.req.map(_.ready := !io.write.valid) 779// 780// val srams = (0 until PortNumber) map { bank => 781// val sramBank = Module(new SRAMTemplate( 782// gen, 783// set = nSets / 2, 784// way = pWay, 785// shouldReset = true, 786// holdRead = true, 787// singlePort = true, 788// withClockGate = true 789// )) 790// 791// sramBank.io.r.req.valid := io.read.req(bank).valid 792// sramBank.io.r.req.bits.apply(setIdx = io.read.req(bank).bits.ridx) 793// 794// if (bank == 0) sramBank.io.w.req.valid := io.write.valid && !io.write.bits.wbankidx 795// else sramBank.io.w.req.valid := io.write.valid && io.write.bits.wbankidx 796// sramBank.io.w.req.bits.apply( 797// data = io.write.bits.wdata, 798// setIdx = io.write.bits.widx, 799// waymask = io.write.bits.wmask.asUInt 800// ) 801// 802// sramBank 803// } 804// 805// io.read.req.map(_.ready := !io.write.valid && srams.map(_.io.r.req.ready).reduce(_ && _)) 806// 807// io.read.resp.rdata := VecInit(srams.map(bank => bank.io.r.resp.asTypeOf(Vec(pWay, gen)))) 808// 809//} 810 811class SRAMTemplateWithFixedWidthIO[T <: Data](gen: T, set: Int, way: Int) extends Bundle { 812 val r: SRAMReadBus[T] = Flipped(new SRAMReadBus(gen, set, way)) 813 val w: SRAMWriteBus[T] = Flipped(new SRAMWriteBus(gen, set, way)) 814} 815 816// Automatically partition the SRAM based on the width of the data and the desired width. 817// final SRAM width = width * way 818class SRAMTemplateWithFixedWidth[T <: Data]( 819 gen: T, 820 set: Int, 821 width: Int, 822 way: Int = 1, 823 shouldReset: Boolean = false, 824 holdRead: Boolean = false, 825 singlePort: Boolean = false, 826 bypassWrite: Boolean = false, 827 withClockGate: Boolean = false 828) extends Module { 829 830 private val dataBits = gen.getWidth 831 private val bankNum = math.ceil(dataBits.toDouble / width.toDouble).toInt 832 private val totalBits = bankNum * width 833 834 val io: SRAMTemplateWithFixedWidthIO[T] = IO(new SRAMTemplateWithFixedWidthIO(gen, set, way)) 835 836 private val wordType = UInt(width.W) 837 private val writeDatas = (0 until bankNum).map { bank => 838 VecInit((0 until way).map { i => 839 io.w.req.bits.data(i).asTypeOf(UInt(totalBits.W)).asTypeOf(Vec(bankNum, wordType))(bank) 840 }) 841 } 842 843 private val srams = (0 until bankNum) map { bank => 844 val sramBank = Module(new SRAMTemplate( 845 wordType, 846 set = set, 847 way = way, 848 shouldReset = shouldReset, 849 holdRead = holdRead, 850 singlePort = singlePort, 851 bypassWrite = bypassWrite, 852 withClockGate = withClockGate 853 )) 854 // read req 855 sramBank.io.r.req.valid := io.r.req.valid 856 sramBank.io.r.req.bits.setIdx := io.r.req.bits.setIdx 857 858 // write req 859 sramBank.io.w.req.valid := io.w.req.valid 860 sramBank.io.w.req.bits.setIdx := io.w.req.bits.setIdx 861 sramBank.io.w.req.bits.data := writeDatas(bank) 862 sramBank.io.w.req.bits.waymask.foreach(_ := io.w.req.bits.waymask.get) 863 864 sramBank 865 } 866 867 io.r.req.ready := !io.w.req.valid 868 (0 until way).foreach { i => 869 io.r.resp.data(i) := VecInit((0 until bankNum).map(bank => 870 srams(bank).io.r.resp.data(i) 871 )).asTypeOf(UInt(totalBits.W))(dataBits - 1, 0).asTypeOf(gen.cloneType) 872 } 873 874 io.r.req.ready := srams.head.io.r.req.ready 875 io.w.req.ready := srams.head.io.w.req.ready 876} 877