1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import chisel3._ 20import chisel3.util._ 21import difftest._ 22import freechips.rocketchip.diplomacy.IdRange 23import freechips.rocketchip.tilelink._ 24import freechips.rocketchip.tilelink.ClientStates._ 25import freechips.rocketchip.tilelink.TLPermissions._ 26import org.chipsalliance.cde.config.Parameters 27import utility._ 28import utils._ 29import xiangshan._ 30import xiangshan.cache._ 31 32abstract class ICacheMissUnitModule(implicit p: Parameters) extends XSModule 33 with HasICacheParameters 34 35abstract class ICacheMissUnitBundle(implicit p: Parameters) extends XSBundle 36 with HasICacheParameters 37 38class Demultiplexer[T <: Data](val gen: T, val n: Int) extends Module { 39 40 /** Hardware module that is used to sequence 1 producers into n consumer. 41 * Priority is given to lower producer. 42 */ 43 require(n >= 2) 44 val io = IO(new Bundle { 45 val in = Flipped(DecoupledIO(gen)) 46 val out = Vec(n, DecoupledIO(gen)) 47 val chosen = Output(UInt(log2Ceil(n).W)) 48 }) 49 50 val grant = false.B +: (1 until n).map(i => (0 until i).map(io.out(_).ready).reduce(_ || _)) 51 for (i <- 0 until n) { 52 io.out(i).bits := io.in.bits 53 io.out(i).valid := !grant(i) && io.in.valid 54 } 55 56 io.in.ready := grant.last || io.out.last.ready 57 io.chosen := PriorityEncoder(VecInit(io.out.map(_.ready))) 58} 59 60class MuxBundle[T <: Data](val gen: T, val n: Int) extends Module { 61 require(n >= 2) 62 val io = IO(new Bundle { 63 val sel = Input(UInt(log2Ceil(n).W)) 64 val in = Flipped(Vec(n, DecoupledIO(gen))) 65 val out = DecoupledIO(gen) 66 }) 67 68 io.in <> DontCare 69 io.out <> DontCare 70 for (i <- 0 until n) { 71 when(io.sel === i.U) { 72 io.out <> io.in(i) 73 } 74 io.in(i).ready := (io.sel === i.U) && io.out.ready 75 } 76} 77 78class ICacheMissReq(implicit p: Parameters) extends ICacheBundle { 79 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 80 val vSetIdx = UInt(idxBits.W) 81} 82 83class ICacheMissResp(implicit p: Parameters) extends ICacheBundle { 84 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 85 val vSetIdx = UInt(idxBits.W) 86 val waymask = UInt(nWays.W) 87 val data = UInt(blockBits.W) 88 val corrupt = Bool() 89} 90 91class LookUpMSHR(implicit p: Parameters) extends ICacheBundle { 92 val info = ValidIO(new ICacheMissReq) 93 val hit = Input(Bool()) 94} 95 96class MSHRResp(implicit p: Parameters) extends ICacheBundle { 97 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 98 val vSetIdx = UInt(idxBits.W) 99 val waymask = UInt(log2Ceil(nWays).W) 100} 101 102class MSHRAcquire(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheBundle { 103 val acquire = new TLBundleA(edge.bundle) 104 val vSetIdx = UInt(idxBits.W) 105} 106 107class ICacheMSHR(edge: TLEdgeOut, isFetch: Boolean, ID: Int)(implicit p: Parameters) extends ICacheMissUnitModule { 108 val io = IO(new Bundle { 109 val fencei = Input(Bool()) 110 val flush = Input(Bool()) 111 val invalid = Input(Bool()) 112 val req = Flipped(DecoupledIO(new ICacheMissReq)) 113 val acquire = DecoupledIO(new MSHRAcquire(edge)) 114 val lookUps = Flipped(Vec(2, new LookUpMSHR)) 115 val resp = ValidIO(new MSHRResp) 116 val victimWay = Input(UInt(log2Ceil(nWays).W)) 117 }) 118 119 val valid = RegInit(Bool(), false.B) 120 // this MSHR doesn't respones to fetch and sram 121 val flush = RegInit(Bool(), false.B) 122 val fencei = RegInit(Bool(), false.B) 123 // this MSHR has been issued 124 val issue = RegInit(Bool(), false.B) 125 126 val blkPaddr = RegInit(UInt((PAddrBits - blockOffBits).W), 0.U) 127 val vSetIdx = RegInit(UInt(idxBits.W), 0.U) 128 val waymask = RegInit(UInt(log2Ceil(nWays).W), 0.U) 129 130 // look up and return result at the same cycle 131 val hits = io.lookUps.map(lookup => 132 valid && !fencei && !flush && (lookup.info.bits.vSetIdx === vSetIdx) && 133 (lookup.info.bits.blkPaddr === blkPaddr) 134 ) 135 // Decoupling valid and bits 136 (0 until 2).foreach(i => io.lookUps(i).hit := hits(i)) 137 138 // disable wake up when hit MSHR (fencei is low) 139 // when(hit) { 140 // flush := false.B 141 // } 142 143 // invalid when the req hasn't been issued 144 when(io.fencei || io.flush) { 145 fencei := true.B 146 flush := true.B 147 when(!issue) { 148 valid := false.B 149 } 150 } 151 152 // receive request and register 153 io.req.ready := !valid && !io.flush && !io.fencei 154 when(io.req.fire) { 155 valid := true.B 156 flush := false.B 157 issue := false.B 158 fencei := false.B 159 blkPaddr := io.req.bits.blkPaddr 160 vSetIdx := io.req.bits.vSetIdx 161 } 162 163 // send request to L2 164 io.acquire.valid := valid && !issue && !io.flush && !io.fencei 165 val getBlock = edge.Get( 166 fromSource = ID.U, 167 toAddress = Cat(blkPaddr, 0.U(blockOffBits.W)), 168 lgSize = log2Up(cacheParams.blockBytes).U 169 )._2 170 io.acquire.bits.acquire := getBlock 171 io.acquire.bits.acquire.user.lift(ReqSourceKey).foreach(_ := MemReqSource.CPUInst.id.U) 172 io.acquire.bits.vSetIdx := vSetIdx 173 174 // get victim way when acquire fire 175 when(io.acquire.fire) { 176 issue := true.B 177 waymask := io.victimWay 178 } 179 180 // invalid request when grant finish 181 when(io.invalid) { 182 valid := false.B 183 } 184 185 // offer the information other than data for write sram and response fetch 186 io.resp.valid := valid && (!flush && !fencei) 187 io.resp.bits.blkPaddr := blkPaddr 188 io.resp.bits.vSetIdx := vSetIdx 189 io.resp.bits.waymask := waymask 190} 191 192class ICacheMissBundle(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheBundle { 193 // difftest 194 val hartId = Input(Bool()) 195 // control 196 val fencei = Input(Bool()) 197 val flush = Input(Bool()) 198 // fetch 199 val fetch_req = Flipped(DecoupledIO(new ICacheMissReq)) 200 val fetch_resp = ValidIO(new ICacheMissResp) 201 // prefetch 202 val prefetch_req = Flipped(DecoupledIO(new ICacheMissReq)) 203 // SRAM Write Req 204 val meta_write = DecoupledIO(new ICacheMetaWriteBundle) 205 val data_write = DecoupledIO(new ICacheDataWriteBundle) 206 // get victim from replacer 207 val victim = new ReplacerVictim 208 // Tilelink 209 val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 210 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 211} 212 213class ICacheMissUnit(edge: TLEdgeOut)(implicit p: Parameters) extends ICacheMissUnitModule { 214 val io = IO(new ICacheMissBundle(edge)) 215 216 /** 217 ****************************************************************************** 218 * fetch have higher priority 219 * fetch MSHR: lower index have a higher priority 220 * prefetch MSHR: the prefetchMSHRs earlier have a higher priority 221 * --------- -------------- ----------- 222 * ---fetch reg--->| Demux |-----> | fetch MSHR |------>| Arbiter |---acquire---> 223 * --------- -------------- ----------- 224 * | fetch MSHR | ^ 225 * -------------- | 226 * | 227 * ----------------- | 228 * | prefetch MSHR | | 229 * --------- ----------------- ----------- 230 * ---fetch reg--->| Demux |----> | prefetch MSHR |---->| Arbiter | 231 * --------- ----------------- ----------- 232 * | ....... | 233 * ----------------- 234 ****************************************************************************** 235 */ 236 237 val fetchDemux = Module(new Demultiplexer(new ICacheMissReq, nFetchMshr)) 238 val prefetchDemux = Module(new Demultiplexer(new ICacheMissReq, nPrefetchMshr)) 239 val prefetchArb = Module(new MuxBundle(new MSHRAcquire(edge), nPrefetchMshr)) 240 val acquireArb = Module(new Arbiter(new MSHRAcquire(edge), nFetchMshr + 1)) 241 242 // To avoid duplicate request reception. 243 val fetchHit, prefetchHit = Wire(Bool()) 244 fetchDemux.io.in <> io.fetch_req 245 fetchDemux.io.in.valid := io.fetch_req.valid && !fetchHit 246 io.fetch_req.ready := fetchDemux.io.in.ready || fetchHit 247 prefetchDemux.io.in <> io.prefetch_req 248 prefetchDemux.io.in.valid := io.prefetch_req.valid && !prefetchHit 249 io.prefetch_req.ready := prefetchDemux.io.in.ready || prefetchHit 250 acquireArb.io.in.last <> prefetchArb.io.out 251 252 // mem_acquire connect 253 io.mem_acquire.valid := acquireArb.io.out.valid 254 io.mem_acquire.bits := acquireArb.io.out.bits.acquire 255 acquireArb.io.out.ready := io.mem_acquire.ready 256 257 val fetchMSHRs = (0 until nFetchMshr).map { i => 258 val mshr = Module(new ICacheMSHR(edge, true, i)) 259 mshr.io.flush := false.B 260 mshr.io.fencei := io.fencei 261 mshr.io.req <> fetchDemux.io.out(i) 262 mshr.io.lookUps(0).info.valid := io.fetch_req.valid 263 mshr.io.lookUps(0).info.bits := io.fetch_req.bits 264 mshr.io.lookUps(1).info.valid := io.prefetch_req.valid 265 mshr.io.lookUps(1).info.bits := io.prefetch_req.bits 266 mshr.io.victimWay := io.victim.way 267 acquireArb.io.in(i) <> mshr.io.acquire 268 mshr 269 } 270 271 val prefetchMSHRs = (0 until nPrefetchMshr).map { i => 272 val mshr = Module(new ICacheMSHR(edge, false, nFetchMshr + i)) 273 mshr.io.flush := io.flush 274 mshr.io.fencei := io.fencei 275 mshr.io.req <> prefetchDemux.io.out(i) 276 mshr.io.lookUps(0).info.valid := io.fetch_req.valid 277 mshr.io.lookUps(0).info.bits := io.fetch_req.bits 278 mshr.io.lookUps(1).info.valid := io.prefetch_req.valid 279 mshr.io.lookUps(1).info.bits := io.prefetch_req.bits 280 mshr.io.victimWay := io.victim.way 281 prefetchArb.io.in(i) <> mshr.io.acquire 282 mshr 283 } 284 285 /** 286 ****************************************************************************** 287 * MSHR look up 288 * - look up all mshr 289 ****************************************************************************** 290 */ 291 val allMSHRs = fetchMSHRs ++ prefetchMSHRs 292 val prefetchHitFetchReq = (io.prefetch_req.bits.blkPaddr === io.fetch_req.bits.blkPaddr) && 293 (io.prefetch_req.bits.vSetIdx === io.fetch_req.bits.vSetIdx) && 294 io.fetch_req.valid 295 fetchHit := allMSHRs.map(mshr => mshr.io.lookUps(0).hit).reduce(_ || _) 296 prefetchHit := allMSHRs.map(mshr => mshr.io.lookUps(1).hit).reduce(_ || _) || prefetchHitFetchReq 297 298 /** 299 ****************************************************************************** 300 * prefetchMSHRs priority 301 * - The requests that enter the prefetchMSHRs earlier have a higher priority in issuing. 302 * - The order of enqueuing is recorded in FIFO when requset enters MSHRs. 303 * - The requests are dispatched in the order they are recorded in FIFO. 304 ****************************************************************************** 305 */ 306 // When the FIFO is full, enqueue and dequeue operations do not occur at the same cycle. 307 // So the depth of the FIFO is set to match the number of MSHRs. 308 // val priorityFIFO = Module(new Queue(UInt(log2Ceil(nPrefetchMshr).W), nPrefetchMshr, hasFlush=true)) 309 val priorityFIFO = Module(new FIFOReg(UInt(log2Ceil(nPrefetchMshr).W), nPrefetchMshr, hasFlush = true)) 310 priorityFIFO.io.flush.get := io.flush || io.fencei 311 priorityFIFO.io.enq.valid := prefetchDemux.io.in.fire 312 priorityFIFO.io.enq.bits := prefetchDemux.io.chosen 313 priorityFIFO.io.deq.ready := prefetchArb.io.out.fire 314 prefetchArb.io.sel := priorityFIFO.io.deq.bits 315 assert( 316 !(priorityFIFO.io.enq.fire ^ prefetchDemux.io.in.fire), 317 "priorityFIFO.io.enq and io.prefetch_req must fire at the same cycle" 318 ) 319 assert( 320 !(priorityFIFO.io.deq.fire ^ prefetchArb.io.out.fire), 321 "priorityFIFO.io.deq and prefetchArb.io.out must fire at the same cycle" 322 ) 323 324 /** 325 ****************************************************************************** 326 * Tilelink D channel (grant) 327 ****************************************************************************** 328 */ 329 // cacheline register 330 val readBeatCnt = RegInit(UInt(log2Up(refillCycles).W), 0.U) 331 val respDataReg = RegInit(VecInit(Seq.fill(refillCycles)(0.U(beatBits.W)))) 332 333 val wait_last = readBeatCnt === (refillCycles - 1).U 334 when(io.mem_grant.fire && edge.hasData(io.mem_grant.bits)) { 335 respDataReg(readBeatCnt) := io.mem_grant.bits.data 336 readBeatCnt := Mux(wait_last, 0.U, readBeatCnt + 1.U) 337 } 338 339 // last transition finsh or corrupt 340 val last_fire = io.mem_grant.fire && edge.hasData(io.mem_grant.bits) && wait_last 341 342 val (_, _, refill_done, _) = edge.addr_inc(io.mem_grant) 343 assert(!(refill_done ^ last_fire), "refill not done!") 344 io.mem_grant.ready := true.B 345 346 val last_fire_r = RegNext(last_fire) 347 val id_r = RegNext(io.mem_grant.bits.source) 348 349 // if any beat is corrupt, the whole response (to mainPipe/metaArray/dataArray) is corrupt 350 val corrupt_r = RegInit(false.B) 351 when(io.mem_grant.fire && edge.hasData(io.mem_grant.bits) && io.mem_grant.bits.corrupt) { 352 corrupt_r := true.B 353 }.elsewhen(io.fetch_resp.fire) { 354 corrupt_r := false.B 355 } 356 357 /** 358 ****************************************************************************** 359 * invalid mshr when finish transition 360 ****************************************************************************** 361 */ 362 (0 until (nFetchMshr + nPrefetchMshr)).foreach(i => allMSHRs(i).io.invalid := last_fire_r && (id_r === i.U)) 363 364 /** 365 ****************************************************************************** 366 * response fetch and write SRAM 367 ****************************************************************************** 368 */ 369 // get request information from MSHRs 370 val allMSHRs_resp = VecInit(allMSHRs.map(mshr => mshr.io.resp)) 371 val mshr_resp = allMSHRs_resp(id_r) 372 373 // get waymask from replacer when acquire fire 374 io.victim.vSetIdx.valid := acquireArb.io.out.fire 375 io.victim.vSetIdx.bits := acquireArb.io.out.bits.vSetIdx 376 val waymask = UIntToOH(mshr_resp.bits.waymask) 377 // NOTE: when flush/fencei, missUnit will still send response to mainPipe/prefetchPipe 378 // this is intentional to fix timing (io.flush -> mainPipe/prefetchPipe s2_miss -> s2_ready -> ftq ready) 379 // unnecessary response will be dropped by mainPipe/prefetchPipe/wayLookup since their sx_valid is set to false 380 val fetch_resp_valid = mshr_resp.valid && last_fire_r 381 // NOTE: but we should not write meta/dataArray when flush/fencei 382 val write_sram_valid = fetch_resp_valid && !corrupt_r && !io.flush && !io.fencei 383 384 // write SRAM 385 io.meta_write.bits.generate( 386 tag = getPhyTagFromBlk(mshr_resp.bits.blkPaddr), 387 idx = mshr_resp.bits.vSetIdx, 388 waymask = waymask, 389 bankIdx = mshr_resp.bits.vSetIdx(0) 390 ) 391 io.data_write.bits.generate( 392 data = respDataReg.asUInt, 393 idx = mshr_resp.bits.vSetIdx, 394 waymask = waymask, 395 bankIdx = mshr_resp.bits.vSetIdx(0) 396 ) 397 398 io.meta_write.valid := write_sram_valid 399 io.data_write.valid := write_sram_valid 400 401 // response fetch 402 io.fetch_resp.valid := fetch_resp_valid 403 io.fetch_resp.bits.blkPaddr := mshr_resp.bits.blkPaddr 404 io.fetch_resp.bits.vSetIdx := mshr_resp.bits.vSetIdx 405 io.fetch_resp.bits.waymask := waymask 406 io.fetch_resp.bits.data := respDataReg.asUInt 407 io.fetch_resp.bits.corrupt := corrupt_r 408 409 /** 410 ****************************************************************************** 411 * performance counter 412 ****************************************************************************** 413 */ 414 // Duplicate requests will be excluded. 415 XSPerfAccumulate("enq_fetch_req", fetchDemux.io.in.fire) 416 XSPerfAccumulate("enq_prefetch_req", prefetchDemux.io.in.fire) 417 418 /** 419 ****************************************************************************** 420 * ChiselDB: record ICache SRAM write log 421 ****************************************************************************** 422 */ 423 class ICacheSRAMDB(implicit p: Parameters) extends ICacheBundle { 424 val blkPaddr = UInt((PAddrBits - blockOffBits).W) 425 val vSetIdx = UInt(idxBits.W) 426 val waymask = UInt(log2Ceil(nWays).W) 427 } 428 429 val isWriteICacheSRAMTable = 430 WireInit(Constantin.createRecord("isWriteICacheSRAMTable" + p(XSCoreParamsKey).HartId.toString)) 431 val ICacheSRAMTable = ChiselDB.createTable("ICacheSRAMTable" + p(XSCoreParamsKey).HartId.toString, new ICacheSRAMDB) 432 433 val ICacheSRAMDBDumpData = Wire(new ICacheSRAMDB) 434 ICacheSRAMDBDumpData.blkPaddr := mshr_resp.bits.blkPaddr 435 ICacheSRAMDBDumpData.vSetIdx := mshr_resp.bits.vSetIdx 436 ICacheSRAMDBDumpData.waymask := OHToUInt(waymask) 437 ICacheSRAMTable.log( 438 data = ICacheSRAMDBDumpData, 439 en = write_sram_valid, 440 clock = clock, 441 reset = reset 442 ) 443 444 /** 445 ****************************************************************************** 446 * Difftest 447 ****************************************************************************** 448 */ 449 if (env.EnableDifftest) { 450 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 451 difftest.coreid := io.hartId 452 difftest.index := 0.U 453 difftest.valid := write_sram_valid 454 difftest.addr := Cat(mshr_resp.bits.blkPaddr, 0.U(blockOffBits.W)) 455 difftest.data := respDataReg.asTypeOf(difftest.data) 456 difftest.idtfr := DontCare 457 } 458} 459