1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import freechips.rocketchip.diplomacy.IdRange 23import freechips.rocketchip.tilelink.ClientStates._ 24import freechips.rocketchip.tilelink.TLPermissions._ 25import freechips.rocketchip.tilelink._ 26import xiangshan._ 27import xiangshan.cache._ 28import utils._ 29import utility._ 30import difftest._ 31 32 33abstract class ICacheMissUnitModule(implicit p: Parameters) extends XSModule 34 with HasICacheParameters 35 36abstract class ICacheMissUnitBundle(implicit p: Parameters) extends XSBundle 37 with HasICacheParameters 38 39 40class Demultiplexer[T <: Data](val gen: T, val n: Int) extends Module 41{ 42 /** Hardware module that is used to sequence 1 producers into n consumer. 43 * Priority is given to lower producer. 44 */ 45 require(n >= 2) 46 val io = IO(new Bundle { 47 val in = Flipped(DecoupledIO(gen)) 48 val out = Vec(n, DecoupledIO(gen)) 49 val chosen = Output(UInt(log2Ceil(n).W)) 50 }) 51 52 val grant = false.B +: (1 until n).map(i=> (0 until i).map(io.out(_).ready).reduce(_||_)) 53 for (i <- 0 until n) { 54 io.out(i).bits := io.in.bits 55 io.out(i).valid := !grant(i) && io.in.valid 56 } 57 58 io.in.ready := grant.last || io.out.last.ready 59 io.chosen := PriorityEncoder(VecInit(io.out.map(_.ready))) 60} 61 62 63class MuxBundle[T <: Data](val gen: T, val n: Int) extends Module 64{ 65 require(n >= 2) 66 val io = IO(new Bundle { 67 val sel = Input(UInt(log2Ceil(n).W)) 68 val in = Flipped(Vec(n, DecoupledIO(gen))) 69 val out = DecoupledIO(gen) 70 }) 71 72 io.in <> DontCare 73 io.out <> DontCare 74 for (i <- 0 until n) { 75 when(io.sel === i.U) { 76 io.out <> io.in(i) 77 } 78 io.in(i).ready := (io.sel === i.U) && io.out.ready 79 } 80} 81 82 83class ICacheMissReq(implicit p: Parameters) extends ICacheBundle { 84 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 85 val vSetIdx = UInt(idxBits.W) 86} 87 88 89class ICacheMissResp(implicit p: Parameters) extends ICacheBundle { 90 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 91 val vSetIdx = UInt(idxBits.W) 92 val waymask = UInt(nWays.W) 93 val data = UInt(blockBits.W) 94 val corrupt = Bool() 95} 96 97 98class LookUpMSHR(implicit p: Parameters) extends ICacheBundle { 99 val info = ValidIO(new ICacheMissReq) 100 val hit = Input(Bool()) 101} 102 103 104class MSHRResp(implicit p: Parameters) extends ICacheBundle { 105 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 106 val vSetIdx = UInt(idxBits.W) 107 val waymask = UInt(log2Ceil(nWays).W) 108} 109 110 111class MSHRAcquire(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheBundle { 112 val acquire = new TLBundleA(edge.bundle) 113 val vSetIdx = UInt(idxBits.W) 114} 115 116class ICacheMSHR(edge: TLEdgeOut, isFetch: Boolean, ID: Int)(implicit p: Parameters) extends ICacheMissUnitModule { 117 val io = IO(new Bundle { 118 val fencei = Input(Bool()) 119 val flush = Input(Bool()) 120 val invalid = Input(Bool()) 121 val req = Flipped(DecoupledIO(new ICacheMissReq)) 122 val acquire = DecoupledIO(new MSHRAcquire(edge)) 123 val lookUps = Flipped(Vec(2, new LookUpMSHR)) 124 val resp = ValidIO(new MSHRResp) 125 val victimWay = Input(UInt(log2Ceil(nWays).W)) 126 }) 127 128 val valid = RegInit(Bool(), false.B) 129 // this MSHR doesn't respones to fetch and sram 130 val flush = RegInit(Bool(), false.B) 131 val fencei = RegInit(Bool(), false.B) 132 // this MSHR has been issued 133 val issue = RegInit(Bool(), false.B) 134 135 val blkPaddr = RegInit(UInt((PAddrBits - blockOffBits).W), 0.U) 136 val vSetIdx = RegInit(UInt(idxBits.W), 0.U) 137 val waymask = RegInit(UInt(log2Ceil(nWays).W), 0.U) 138 139 // look up and return result at the same cycle 140 val hits = io.lookUps.map(lookup => valid && !fencei && !flush && (lookup.info.bits.vSetIdx === vSetIdx) && 141 (lookup.info.bits.blkPaddr === blkPaddr)) 142 // Decoupling valid and bits 143 (0 until 2).foreach { i => 144 io.lookUps(i).hit := hits(i) 145 } 146 147 // disable wake up when hit MSHR (fencei is low) 148 // when(hit) { 149 // flush := false.B 150 // } 151 152 // invalid when the req hasn't been issued 153 when(io.fencei || io.flush) { 154 fencei := true.B 155 flush := true.B 156 when(!issue) { 157 valid := false.B 158 } 159 } 160 161 // receive request and register 162 io.req.ready := !valid && !io.flush && !io.fencei 163 when(io.req.fire) { 164 valid := true.B 165 flush := false.B 166 issue := false.B 167 fencei := false.B 168 blkPaddr := io.req.bits.blkPaddr 169 vSetIdx := io.req.bits.vSetIdx 170 } 171 172 // send request to L2 173 io.acquire.valid := valid && !issue && !io.flush && !io.fencei 174 val getBlock = edge.Get( 175 fromSource = ID.U, 176 toAddress = Cat(blkPaddr, 0.U(blockOffBits.W)), 177 lgSize = (log2Up(cacheParams.blockBytes)).U 178 )._2 179 io.acquire.bits.acquire := getBlock 180 io.acquire.bits.acquire.user.lift(ReqSourceKey).foreach(_ := MemReqSource.CPUInst.id.U) 181 io.acquire.bits.vSetIdx := vSetIdx 182 183 // get victim way when acquire fire 184 when(io.acquire.fire) { 185 issue := true.B 186 waymask := io.victimWay 187 } 188 189 // invalid request when grant finish 190 when(io.invalid) { 191 valid := false.B 192 } 193 194 // offer the information other than data for write sram and response fetch 195 io.resp.valid := valid && (!flush && !fencei) 196 io.resp.bits.blkPaddr := blkPaddr 197 io.resp.bits.vSetIdx := vSetIdx 198 io.resp.bits.waymask := waymask 199} 200 201class ICacheMissBundle(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheBundle{ 202 // difftest 203 val hartId = Input(Bool()) 204 // control 205 val fencei = Input(Bool()) 206 val flush = Input(Bool()) 207 // fetch 208 val fetch_req = Flipped(DecoupledIO(new ICacheMissReq)) 209 val fetch_resp = ValidIO(new ICacheMissResp) 210 // prefetch 211 val prefetch_req = Flipped(DecoupledIO(new ICacheMissReq)) 212 // SRAM Write Req 213 val meta_write = DecoupledIO(new ICacheMetaWriteBundle) 214 val data_write = DecoupledIO(new ICacheDataWriteBundle) 215 // get victim from replacer 216 val victim = new ReplacerVictim 217 // Tilelink 218 val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 219 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 220} 221 222 223class ICacheMissUnit(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheMissUnitModule 224{ 225 val io = IO(new ICacheMissBundle(edge)) 226 227 /** 228 ****************************************************************************** 229 * fetch have higher priority 230 * fetch MSHR: lower index have a higher priority 231 * prefetch MSHR: the prefetchMSHRs earlier have a higher priority 232 * --------- -------------- ----------- 233 * ---fetch reg--->| Demux |-----> | fetch MSHR |------>| Arbiter |---acquire---> 234 * --------- -------------- ----------- 235 * | fetch MSHR | ^ 236 * -------------- | 237 * | 238 * ----------------- | 239 * | prefetch MSHR | | 240 * --------- ----------------- ----------- 241 * ---fetch reg--->| Demux |----> | prefetch MSHR |---->| Arbiter | 242 * --------- ----------------- ----------- 243 * | ....... | 244 * ----------------- 245 ****************************************************************************** 246 */ 247 248 val fetchDemux = Module(new Demultiplexer(new ICacheMissReq, nFetchMshr)) 249 val prefetchDemux = Module(new Demultiplexer(new ICacheMissReq, nPrefetchMshr)) 250 val prefetchArb = Module(new MuxBundle(new MSHRAcquire(edge), nPrefetchMshr)) 251 val acquireArb = Module(new Arbiter(new MSHRAcquire(edge), nFetchMshr + 1)) 252 253 // To avoid duplicate request reception. 254 val fetchHit, prefetchHit = Wire(Bool()) 255 fetchDemux.io.in <> io.fetch_req 256 fetchDemux.io.in.valid := io.fetch_req.valid && !fetchHit 257 io.fetch_req.ready := fetchDemux.io.in.ready || fetchHit 258 prefetchDemux.io.in <> io.prefetch_req 259 prefetchDemux.io.in.valid := io.prefetch_req.valid && !prefetchHit 260 io.prefetch_req.ready := prefetchDemux.io.in.ready || prefetchHit 261 acquireArb.io.in.last <> prefetchArb.io.out 262 263 // mem_acquire connect 264 io.mem_acquire.valid := acquireArb.io.out.valid 265 io.mem_acquire.bits := acquireArb.io.out.bits.acquire 266 acquireArb.io.out.ready := io.mem_acquire.ready 267 268 val fetchMSHRs = (0 until nFetchMshr).map { i => 269 val mshr = Module(new ICacheMSHR(edge, true, i)) 270 mshr.io.flush := false.B 271 mshr.io.fencei := io.fencei 272 mshr.io.req <> fetchDemux.io.out(i) 273 mshr.io.lookUps(0).info.valid := io.fetch_req.valid 274 mshr.io.lookUps(0).info.bits := io.fetch_req.bits 275 mshr.io.lookUps(1).info.valid := io.prefetch_req.valid 276 mshr.io.lookUps(1).info.bits := io.prefetch_req.bits 277 mshr.io.victimWay := io.victim.way 278 acquireArb.io.in(i) <> mshr.io.acquire 279 mshr 280 } 281 282 val prefetchMSHRs = (0 until nPrefetchMshr).map { i => 283 val mshr = Module(new ICacheMSHR(edge, false, nFetchMshr + i)) 284 mshr.io.flush := io.flush 285 mshr.io.fencei := io.fencei 286 mshr.io.req <> prefetchDemux.io.out(i) 287 mshr.io.lookUps(0).info.valid := io.fetch_req.valid 288 mshr.io.lookUps(0).info.bits := io.fetch_req.bits 289 mshr.io.lookUps(1).info.valid := io.prefetch_req.valid 290 mshr.io.lookUps(1).info.bits := io.prefetch_req.bits 291 mshr.io.victimWay := io.victim.way 292 prefetchArb.io.in(i) <> mshr.io.acquire 293 mshr 294 } 295 296 /** 297 ****************************************************************************** 298 * MSHR look up 299 * - look up all mshr 300 ****************************************************************************** 301 */ 302 val allMSHRs = (fetchMSHRs ++ prefetchMSHRs) 303 val prefetchHitFetchReq = (io.prefetch_req.bits.blkPaddr === io.fetch_req.bits.blkPaddr) && 304 (io.prefetch_req.bits.vSetIdx === io.fetch_req.bits.vSetIdx) && 305 io.fetch_req.valid 306 fetchHit := allMSHRs.map(mshr => mshr.io.lookUps(0).hit).reduce(_||_) 307 prefetchHit := allMSHRs.map(mshr => mshr.io.lookUps(1).hit).reduce(_||_) || prefetchHitFetchReq 308 309 /** 310 ****************************************************************************** 311 * prefetchMSHRs priority 312 * - The requests that enter the prefetchMSHRs earlier have a higher priority in issuing. 313 * - The order of enqueuing is recorded in FIFO when requset enters MSHRs. 314 * - The requests are dispatched in the order they are recorded in FIFO. 315 ****************************************************************************** 316 */ 317 // When the FIFO is full, enqueue and dequeue operations do not occur at the same cycle. 318 // So the depth of the FIFO is set to match the number of MSHRs. 319 // val priorityFIFO = Module(new Queue(UInt(log2Ceil(nPrefetchMshr).W), nPrefetchMshr, hasFlush=true)) 320 val priorityFIFO = Module(new FIFOReg(UInt(log2Ceil(nPrefetchMshr).W), nPrefetchMshr, hasFlush=true)) 321 priorityFIFO.io.flush.get := io.flush || io.fencei 322 priorityFIFO.io.enq.valid := prefetchDemux.io.in.fire 323 priorityFIFO.io.enq.bits := prefetchDemux.io.chosen 324 priorityFIFO.io.deq.ready := prefetchArb.io.out.fire 325 prefetchArb.io.sel := priorityFIFO.io.deq.bits 326 assert(!(priorityFIFO.io.enq.fire ^ prefetchDemux.io.in.fire), "priorityFIFO.io.enq and io.prefetch_req must fire at the same cycle") 327 assert(!(priorityFIFO.io.deq.fire ^ prefetchArb.io.out.fire), "priorityFIFO.io.deq and prefetchArb.io.out must fire at the same cycle") 328 329 /** 330 ****************************************************************************** 331 * Tilelink D channel (grant) 332 ****************************************************************************** 333 */ 334 //cacheline register 335 val readBeatCnt = RegInit(UInt(log2Up(refillCycles).W), 0.U) 336 val respDataReg = RegInit(VecInit(Seq.fill(refillCycles)(0.U(beatBits.W)))) 337 338 val wait_last = readBeatCnt === (refillCycles - 1).U 339 when(io.mem_grant.fire && edge.hasData(io.mem_grant.bits)) { 340 respDataReg(readBeatCnt) := io.mem_grant.bits.data 341 readBeatCnt := Mux(wait_last || io.mem_grant.bits.corrupt, 0.U, readBeatCnt + 1.U) 342 } 343 344 // last transition finsh or corrupt 345 val last_fire = io.mem_grant.fire && edge.hasData(io.mem_grant.bits) && 346 (wait_last || io.mem_grant.bits.corrupt) 347 348 val (_, _, refill_done, _) = edge.addr_inc(io.mem_grant) 349 assert(!(refill_done ^ last_fire), "refill not done!") 350 io.mem_grant.ready := true.B 351 352 val last_fire_r = RegNext(last_fire) 353 val id_r = RegNext(io.mem_grant.bits.source) 354 val corrupt_r = RegNext(io.mem_grant.bits.corrupt) 355 356 /** 357 ****************************************************************************** 358 * invalid mshr when finish transition 359 ****************************************************************************** 360 */ 361 (0 until (nFetchMshr + nPrefetchMshr)).foreach{ i => 362 allMSHRs(i).io.invalid := last_fire_r && (id_r === i.U) 363 } 364 365 /** 366 ****************************************************************************** 367 * response fetch and write SRAM 368 ****************************************************************************** 369 */ 370 // get request information from MSHRs 371 val allMSHRs_resp = VecInit(allMSHRs.map(mshr => mshr.io.resp)) 372 val mshr_resp = allMSHRs_resp(id_r) 373 374 // get waymask from replacer when acquire fire 375 io.victim.vSetIdx.valid := acquireArb.io.out.valid 376 io.victim.vSetIdx.bits := acquireArb.io.out.bits.vSetIdx 377 val waymask = UIntToOH(mshr_resp.bits.waymask) 378 val fetch_resp_valid = mshr_resp.valid && last_fire_r && !io.flush && !io.fencei 379 val write_sram_valid = fetch_resp_valid && !corrupt_r 380 381 // write SRAM 382 io.meta_write.bits.generate(tag = getPhyTagFromBlk(mshr_resp.bits.blkPaddr), 383 idx = mshr_resp.bits.vSetIdx, 384 waymask = waymask, 385 bankIdx = mshr_resp.bits.vSetIdx(0)) 386 io.data_write.bits.generate(data = respDataReg.asUInt, 387 idx = mshr_resp.bits.vSetIdx, 388 waymask = waymask, 389 bankIdx = mshr_resp.bits.vSetIdx(0)) 390 391 io.meta_write.valid := write_sram_valid 392 io.data_write.valid := write_sram_valid 393 394 // response fetch 395 io.fetch_resp.valid := fetch_resp_valid 396 io.fetch_resp.bits.blkPaddr := mshr_resp.bits.blkPaddr 397 io.fetch_resp.bits.vSetIdx := mshr_resp.bits.vSetIdx 398 io.fetch_resp.bits.waymask := waymask 399 io.fetch_resp.bits.data := respDataReg.asUInt 400 io.fetch_resp.bits.corrupt := corrupt_r 401 402 /** 403 ****************************************************************************** 404 * performance counter 405 ****************************************************************************** 406 */ 407 // Duplicate requests will be excluded. 408 XSPerfAccumulate("enq_fetch_req", fetchDemux.io.in.fire) 409 XSPerfAccumulate("enq_prefetch_req", prefetchDemux.io.in.fire) 410 411 /** 412 ****************************************************************************** 413 * ChiselDB: record ICache SRAM write log 414 ****************************************************************************** 415 */ 416 class ICacheSRAMDB(implicit p: Parameters) extends ICacheBundle{ 417 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 418 val vSetIdx = UInt(idxBits.W) 419 val waymask = UInt(log2Ceil(nWays).W) 420 } 421 422 val isWriteICacheSRAMTable = WireInit(Constantin.createRecord("isWriteICacheSRAMTable" + p(XSCoreParamsKey).HartId.toString)) 423 val ICacheSRAMTable = ChiselDB.createTable("ICacheSRAMTable" + p(XSCoreParamsKey).HartId.toString, new ICacheSRAMDB) 424 425 val ICacheSRAMDBDumpData = Wire(new ICacheSRAMDB) 426 ICacheSRAMDBDumpData.blkPaddr := mshr_resp.bits.blkPaddr 427 ICacheSRAMDBDumpData.vSetIdx := mshr_resp.bits.vSetIdx 428 ICacheSRAMDBDumpData.waymask := OHToUInt(waymask) 429 ICacheSRAMTable.log( 430 data = ICacheSRAMDBDumpData, 431 en = write_sram_valid, 432 clock = clock, 433 reset = reset 434 ) 435 436 /** 437 ****************************************************************************** 438 * Difftest 439 ****************************************************************************** 440 */ 441 if (env.EnableDifftest) { 442 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 443 difftest.coreid := io.hartId 444 difftest.index := 0.U 445 difftest.valid := write_sram_valid 446 difftest.addr := Cat(mshr_resp.bits.blkPaddr, 0.U(blockOffBits.W)) 447 difftest.data := respDataReg.asTypeOf(difftest.data) 448 difftest.idtfr := DontCare 449 } 450}