1/*************************************************************************************** 2 * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3 * Copyright (c) 2020-2021 Peng Cheng Laboratory 4 * 5 * XiangShan is licensed under Mulan PSL v2. 6 * You can use this software according to the terms and conditions of the Mulan PSL v2. 7 * You may obtain a copy of Mulan PSL v2 at: 8 * http://license.coscl.org.cn/MulanPSL2 9 * 10 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13 * 14 * See the Mulan PSL v2 for more details. 15 ***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import difftest._ 23import freechips.rocketchip.tilelink._ 24import utils._ 25import xiangshan.cache.mmu._ 26import xiangshan.frontend._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28import huancun.PreferCacheKey 29import xiangshan.XSCoreParamsKey 30import utility._ 31 32 33abstract class IPrefetchBundle(implicit p: Parameters) extends ICacheBundle 34abstract class IPrefetchModule(implicit p: Parameters) extends ICacheModule 35 36//TODO: remove this 37object DebugFlags { 38 val fdip = false 39} 40 41class PIQData(implicit p: Parameters) extends IPrefetchBundle { 42 val ptage = UInt(tagBits.W) 43 val vSetIdx = UInt(idxBits.W) 44 val cacheline = UInt(blockBits.W) 45 val writeBack = Bool() 46} 47 48class PIQToMainPipe(implicit p: Parameters) extends IPrefetchBundle{ 49 val info = DecoupledIO(new PIQData) 50} 51/* need change name */ 52class MainPipeMissInfo(implicit p: Parameters) extends IPrefetchBundle { 53 val ptage = UInt(tagBits.W) 54 val vSetIdx = UInt(idxBits.W) 55} 56 57class PrefetchReq(implicit p: Parameters) extends IPrefetchBundle{ 58 val paddr = UInt(PAddrBits.W) 59 val vSetIdx = UInt(idxBits.W) 60} 61 62class PrefetchBufferIO(implicit p: Parameters) extends IPrefetchBundle { 63 val hartId = Input(UInt(8.W)) 64 val fencei = Input(Bool()) 65 val IPFFilterRead = new IPFBufferFilterRead 66 val IPFBufferRead = new IPFBufferRead 67 val IPFBufferWrite = Flipped(DecoupledIO(new IPFBufferWrite)) 68 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 69 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 70 val IPFReplacer = new IPFReplacer 71} 72 73class PrefetchBuffer(implicit p: Parameters) extends IPrefetchModule 74{ 75 val io = IO(new PrefetchBufferIO) 76 77 val fromIPrefetch = io.IPFFilterRead.req 78 val toIPrefetch = io.IPFFilterRead.resp 79 val fromMainPipe = io.IPFBufferRead.req 80 val toMainPipe = io.IPFBufferRead.resp 81 val fromPIQ = io.IPFBufferWrite 82 val toICacheMeta = io.metaWrite 83 val toICacheData = io.dataWrite 84 val fromReplacer = io.IPFReplacer 85 86 class IPFBufferEntryMeta(implicit p: Parameters) extends IPrefetchBundle 87 { 88 val valid = Bool() 89 val paddr = UInt(PAddrBits.W) 90 val vSetIdx = UInt(idxBits.W) 91 val confidence = UInt(log2Ceil(maxIPFMoveConf + 1).W) 92 } 93 94 class IPFBufferEntryData(implicit p: Parameters) extends IPrefetchBundle 95 { 96 val cacheline = UInt(blockBits.W) 97 } 98 99 def InitQueue[T <: Data](entry: T, size: Int): Vec[T] ={ 100 return RegInit(VecInit(Seq.fill(size)(0.U.asTypeOf(entry.cloneType)))) 101 } 102 103 val meta_buffer = InitQueue(new IPFBufferEntryMeta, size = nIPFBufferSize) 104 val data_buffer = InitQueue(new IPFBufferEntryData, size = nIPFBufferSize) 105 106 /** 107 ****************************************************************************** 108 * handle look-up request from IPrefetchPipe 109 * - 1. Receive request from IPrefetch 110 * - 2. Look up whether hit in meta_buffer 111 * - 3. Send response to IPrefetch at the same cycle 112 ****************************************************************************** 113 */ 114 val fr_block = get_block(fromIPrefetch.paddr) 115 val fr_hit_oh = meta_buffer.map(e => e.valid && (get_block(e.paddr) === fr_block)) 116 toIPrefetch.ipf_hit := ParallelOR(fr_hit_oh) 117 // only hit one entry in meta_buffer at the same time 118 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 prefetch buffer for filter") 119 120 /** 121 ****************************************************************************** 122 * handle read request from ICacheMainPipe 123 * - 1. Receive request from ICacheMainPipe 124 * - 2. Look up whether hit in meta_buffer and get hitted data 125 * - 3. Send response to ICacheMainPipe at the same cycle 126 ****************************************************************************** 127 */ 128 fromMainPipe.foreach(_.ready := DontCare) 129 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 130 val r_hit_oh = r_blocks.map (ptag => meta_buffer.map(e => e.valid && (get_block(e.paddr) === ptag))) 131 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 132 val r_hit_data = r_hit_oh.map (oh => Mux1H(oh, data_buffer.map(_.cacheline))) 133 val r_hit = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 134 (0 until PortNumber).foreach (i => { 135 toMainPipe(i).ipf_hit := r_hit(i) 136 toMainPipe(i).cacheline := r_hit_data(i) 137 // only hit one entry in meta_buffer 138 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 prefetch buffer") 139 XSPerfAccumulate("fdip_mainpipe_hit_in_ipf_" + i, r_hit(i)) 140 }) 141 142 143 /** 144 ****************************************************************************** 145 * move the entry arriving max confidence to icache 146 * - 1. Chose a moved entry from the entries that reach max confidence (LSB first) 147 * - 2. Get victim way of array from replacer of ICacheMainPipe 148 * - 3. Send write req to metaArray and dataArray and ensure fire together 149 ****************************************************************************** 150 */ 151 val move_oh = meta_buffer.map (e => e.confidence === maxIPFMoveConf.U && e.valid) 152 val move_need = ParallelOR(move_oh) 153 val curr_move_ptr = PriorityEncoder(move_oh) 154 155 fromReplacer.vsetIdx := meta_buffer(curr_move_ptr).vSetIdx 156 157 toICacheMeta.valid := move_need 158 toICacheMeta.bits.generate( 159 tag = get_phy_tag(meta_buffer(curr_move_ptr).paddr), 160 idx = meta_buffer(curr_move_ptr).vSetIdx, 161 waymask = fromReplacer.waymask, 162 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0)) 163 164 toICacheData.valid := move_need 165 toICacheData.bits.generate( 166 data = data_buffer(curr_move_ptr).cacheline, 167 idx = meta_buffer(curr_move_ptr).vSetIdx, 168 waymask = fromReplacer.waymask, 169 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0), 170 paddr = meta_buffer(curr_move_ptr).paddr) 171 172 // TODO: how to ensure 173 assert((toICacheMeta.fire && toICacheData.fire) || (!toICacheMeta.fire && !toICacheData.fire), 174 "meta and data array need fire at same time") 175 176 XSPerfAccumulate("fdip_move_to_icache", toICacheMeta.fire && toICacheData.fire) 177 178 when(toICacheMeta.fire) { 179 XSDebug(p"PrefetchBuffer: write prefetch data to icache . vSetIdx:${meta_buffer(curr_move_ptr).vSetIdx} " + 180 p"paddr:${meta_buffer(curr_move_ptr).paddr}") 181 } 182 183 /** 184 ****************************************************************************** 185 * Prefetch buffer write logic 186 * Read old data when read and write occur in the same cycle 187 * There are 4 situations that can cause meta_buffer and data_buffer to be written 188 * - 1. ICacheMainPipe read hit that need to change some status 189 * - 2. Move prefetch cacheline to icache 190 * - 3. Receive write request from ICacheMissUnit 191 * - 4. Receive fencei req that need to flush all buffer 192 * Priority: 4 > 3 > 2 > 1 193 * The order of the code determines the priority, with the following priority being higher 194 ****************************************************************************** 195 */ 196 197 /** 1. confidence++ for every ICacheMainPipe hit */ 198 (0 until PortNumber) .map (i => 199 when(r_hit(i)) { 200 // handle overflow 201 when(meta_buffer(curr_hit_ptr(i)).confidence =/= maxIPFMoveConf.U) { 202 meta_buffer(curr_hit_ptr(i)).confidence := meta_buffer(curr_hit_ptr(i)).confidence + 1.U 203 } 204 } 205 ) 206 207 /** 2. set entry invalid after move to icache */ 208 when(toICacheMeta.fire) { 209 meta_buffer(curr_move_ptr).valid := false.B 210 } 211 212 /** 3. write prefetch data to entry */ 213 // try to get a free entry ptr 214 val free_oh = meta_buffer.map (_.valid === false.B) 215 val has_free = ParallelOR(free_oh) 216 val ptr_from_free = PriorityEncoder(free_oh) 217 // get a ptr from random replacement policy 218 val replacer = ReplacementPolicy.fromString(Some("random"), nIPFBufferSize) 219 // write to free entry if has, otherwise random entry 220 val curr_write_ptr = Mux(has_free, ptr_from_free, replacer.way) 221 fromPIQ.ready := DontCare 222 when(fromPIQ.valid) { 223 meta_buffer(curr_write_ptr).valid := true.B 224 meta_buffer(curr_write_ptr).confidence := fromPIQ.bits.has_hit.asUInt 225 meta_buffer(curr_write_ptr).paddr := fromPIQ.bits.paddr 226 meta_buffer(curr_write_ptr).vSetIdx := fromPIQ.bits.vSetIdx 227 data_buffer(curr_write_ptr).cacheline := fromPIQ.bits.cacheline 228 } 229 230 if(DebugFlags.fdip) { 231 when(fromPIQ.valid) { 232 printf(p"PrefetchBuffer: receive prefetch data. vSetIdx:${fromPIQ.bits.vSetIdx} paddr:${fromPIQ.bits.paddr}") 233 } 234 } 235 236 XSPerfAccumulate("fdip_vitcm_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence =/= 0.U)) 237 XSPerfAccumulate("fdip_vitcm_no_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence === 0.U)) 238 239 /** 4. flush all prefetch buffer when fencei */ 240 when(io.fencei) { 241 meta_buffer.foreach { b => 242 b.valid := false.B 243 b.confidence := 0.U 244 } 245 } 246 247 XSPerfAccumulate("fdip_fencei_cycle", io.fencei) 248 249 if (env.EnableDifftest) { 250 val difftest = DifftestModule(new DiffRefillEvent) 251 difftest.clock := clock 252 difftest.coreid := io.hartId 253 difftest.index := 6.U 254 difftest.valid := toICacheData.fire 255 difftest.addr := toICacheData.bits.paddr 256 difftest.data := toICacheData.bits.data.asTypeOf(difftest.data) 257 } 258} 259 260class IPredfetchIO(implicit p: Parameters) extends IPrefetchBundle { 261 val ftqReq = Flipped(new FtqPrefechBundle) 262 val iTLBInter = new TlbRequestIO 263 val pmp = new ICachePMPBundle 264 val metaReadReq = Decoupled(new ICacheMetaReadReqBundle) 265 val metaReadResp = Input(new ICacheMetaReadRespBundle) 266 val prefetchReq = DecoupledIO(new PrefetchReq) 267 val mshrInfo = Flipped(Vec(PortNumber, ValidIO(UInt(PAddrBits.W)))) 268 val IPFFilterRead = Flipped(new IPFBufferFilterRead) 269 val PIQFilterRead = Flipped(new PIQFilterRead) 270 /** icache main pipe to prefetch pipe */ 271 val mainPipeMissInfo = Flipped(Vec(PortNumber,ValidIO(new MainPipeMissInfo))) 272 val fencei = Input(Bool()) 273} 274 275class IPrefetchPipe(implicit p: Parameters) extends IPrefetchModule 276{ 277 val io = IO(new IPredfetchIO) 278 279 val enableBit = RegInit(false.B) 280 enableBit := false.B 281 282 val fromFtq = io.ftqReq 283 val mainPipeMissInfo = io.mainPipeMissInfo 284 val (toITLB, fromITLB) = (io.iTLBInter.req, io.iTLBInter.resp) 285 val (toIMeta, fromIMeta, fromIMetaValid) = (io.metaReadReq, io.metaReadResp.metaData, io.metaReadResp.entryValid) 286 val (toIPFBuffer, fromIPFBuffer) = (io.IPFFilterRead.req, io.IPFFilterRead.resp) 287 val (toPIQ, fromPIQ) = (io.PIQFilterRead.req, io.PIQFilterRead.resp) 288 val (toPMP, fromPMP) = (io.pmp.req, io.pmp.resp) 289 val fromMSHR = io.mshrInfo 290 val toPIQEnqReq = io.prefetchReq 291 292 val p0_fire, p1_fire, p2_fire = WireInit(false.B) 293 val p0_discard, p1_discard, p2_discard = WireInit(false.B) 294 val p0_ready, p1_ready, p2_ready = WireInit(false.B) 295 296 /** 297 ****************************************************************************** 298 * IPrefetch Stage 0 299 * - 1. send req to IMeta 300 * - 2. send req to ITLB (no blocked) 301 * - 3. check last req 302 ****************************************************************************** 303 */ 304 val p0_valid = fromFtq.req.valid 305 306 val p0_vaddr = addrAlign(fromFtq.req.bits.target, blockBytes, VAddrBits) 307 val p0_vaddr_reg = RegEnable(p0_vaddr, p0_fire) 308 val p0_req_cancel = Wire(Bool()) 309 310 /** 1. send req to IMeta */ 311 toIMeta.valid := p0_valid && !p0_req_cancel 312 toIMeta.bits.idx := get_idx(p0_vaddr) 313 314 /** 2. send req to ITLB (no blocked) */ 315 toITLB.valid := p0_valid && !p0_req_cancel 316 toITLB.bits.size := 3.U // TODO: fix the size 317 toITLB.bits.vaddr := p0_vaddr 318 toITLB.bits.debug.pc := p0_vaddr 319 toITLB.bits.kill := DontCare 320 toITLB.bits.cmd := TlbCmd.exec 321 toITLB.bits.debug.robIdx := DontCare 322 toITLB.bits.debug.isFirstIssue := DontCare 323 toITLB.bits.memidx := DontCare 324 toITLB.bits.no_translate := false.B 325 fromITLB.ready := true.B 326 // TODO: whether to handle tlb miss for prefetch 327 io.iTLBInter.req_kill := false.B 328 329 /** FTQ request port */ 330 fromFtq.req.ready := p0_ready 331 332 /** 3. Check last req: request from FTQ is same as last time or behind */ 333 val p0_hit_behind = Wire(Bool()) 334 335 /** stage 0 control */ 336 // Cancel request when prefetch not enable or the request from FTQ is same as last time 337 p0_req_cancel := !enableBit || p0_hit_behind || io.fencei 338 p0_ready := p0_req_cancel || p1_ready && toITLB.ready && toIMeta.ready 339 p0_fire := p0_valid && p1_ready && toITLB.ready && toIMeta.ready && enableBit && !p0_req_cancel 340 p0_discard := p0_valid && p0_req_cancel 341 342 if(DebugFlags.fdip) { 343 when(p0_discard) { 344 printf(p"IPrefetchPipe: discard req in s0.") 345 } 346 } 347 348 /** 349 ****************************************************************************** 350 * IPrefetch Stage 1 351 * - 1. Receive resp from ITLB (no blocked) 352 * - 2. Receive resp from IMeta 353 * - 3. Check MSHR 354 ****************************************************************************** 355 */ 356 val p1_valid = generatePipeControl(lastFire = p0_fire, thisFire = p1_fire || p1_discard, thisFlush = false.B, lastFlush = false.B) 357 358 val p1_vaddr = RegEnable(p0_vaddr, p0_fire) 359 val p1_req_cancel = Wire(Bool()) 360 361 /** 1. Receive resp from ITLB (no blocked) */ 362 val p1_tlb_resp_miss = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.miss) 363 val p1_tlb_resp_paddr = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.paddr(0)) 364 val p1_tlb_resp_pf = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).pf.instr) 365 val p1_tlb_resp_af = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).af.instr) 366 val p1_exception = VecInit(Seq(p1_tlb_resp_pf, p1_tlb_resp_af)) 367 val p1_has_except = p1_exception.reduce(_ || _) 368 val p1_paddr = p1_tlb_resp_paddr 369 370 /** 2. Receive resp from IMeta. Register the data first because of strict timing. */ 371 val p1_meta_ptags_reg = RegEnable(VecInit(fromIMeta.map(way => way.tag)), RegNext(p0_fire)) 372 val p1_meta_valids_reg = RegEnable(fromIMetaValid, RegNext(p0_fire)) 373 374 /** 3. Check MSHR */ 375 val p1_check_in_mshr = VecInit(fromMSHR.map(mshr => mshr.valid && mshr.bits === addrAlign(p1_paddr, blockBytes, PAddrBits))).reduce(_||_) 376 377 /** Stage 1 control */ 378 p1_req_cancel := p1_check_in_mshr || p1_tlb_resp_miss || p1_has_except || io.fencei 379 p1_ready := p1_valid && p2_ready || !p1_valid 380 p1_fire := p1_valid && !p1_req_cancel && p2_ready && enableBit 381 p1_discard := p1_valid && p1_req_cancel 382 383 when(p1_discard) { 384 XSDebug(p"IPrefetchPipe: discard req in s1. vaddr:${p1_vaddr}") 385 } 386 387 /** 388 ****************************************************************************** 389 * IPrefetch Stage 2 390 * - 1. IMeta Check: check whether req hit in icache 391 * - 2. PMP Check: send req and receive resq in the same cycle 392 * - 3. Check miss handling by main pipe 393 * - 4. Prefetch buffer Check 394 * - 5. Prefetch Queue Check 395 * - 6. MSHR check 396 * - 7. send req to piq 397 ****************************************************************************** 398 */ 399 val p2_valid = generatePipeControl(lastFire = p1_fire, thisFire = p2_fire || p2_discard, thisFlush = false.B, lastFlush = false.B) 400 401 val p2_paddr = RegEnable(p1_paddr, p1_fire) 402 val p2_vaddr = RegEnable(p1_vaddr, p1_fire) 403 val p2_req_cancel = Wire(Bool()) 404 val p2_vidx = get_idx(p2_vaddr) 405 406 p0_hit_behind := (p0_vaddr === p1_vaddr) || (p0_vaddr === p2_vaddr) 407 408 /** 1. IMeta Check */ 409 val p2_ptag = get_phy_tag(p2_paddr) 410 val p2_tag_eq_vec = VecInit(p1_meta_ptags_reg.map(_ === p2_ptag )) 411 val p2_tag_match_vec = VecInit(p2_tag_eq_vec.zipWithIndex.map{ case(way_tag_eq, w) => way_tag_eq && p1_meta_valids_reg(w)}) 412 val p2_tag_match = DataHoldBypass(ParallelOR(p2_tag_match_vec), RegNext(p1_fire)) 413 414 /** 2. PMP Check */ 415 // TODO: When fromPMP.mmio is low but fromPMP.instr is high, which is a except? 416 val p2_exception = DataHoldBypass((fromPMP.mmio && !fromPMP.instr) || fromPMP.instr, RegNext(p1_fire)) 417 // PMP request 418 toPMP.valid := p2_valid 419 toPMP.bits.addr := p2_paddr 420 toPMP.bits.size := 3.U 421 toPMP.bits.cmd := TlbCmd.exec 422 423 /** 3. Check miss handling by main pipe */ 424 val p2_hit_mp_miss = VecInit((0 until PortNumber).map(i => 425 mainPipeMissInfo(i).valid && (mainPipeMissInfo(i).bits.ptage === p2_ptag && 426 (mainPipeMissInfo(i).bits.vSetIdx === p2_vidx)))).reduce(_||_) 427 428 /** 4. Prefetch buffer Check */ 429 toIPFBuffer.paddr := p2_paddr 430 val p2_hit_buffer = fromIPFBuffer.ipf_hit 431 432 /** 5. Prefetch Queue Check */ 433 toPIQ.paddr := p2_paddr 434 val p2_hit_piq = fromPIQ.piq_hit 435 436 /** 6. MSHR check */ 437 val p2_check_in_mshr = VecInit(fromMSHR.map(mshr => mshr.valid && mshr.bits === addrAlign(p2_paddr, blockBytes, PAddrBits))).reduce(_||_) 438 439 /** 7. send req to piq */ 440 toPIQEnqReq.valid := p2_valid && !p2_req_cancel 441 toPIQEnqReq.bits.paddr := p2_paddr 442 toPIQEnqReq.bits.vSetIdx := p2_vidx 443 444 /** Stage 2 control */ 445 p2_req_cancel := p2_tag_match || p2_exception || p2_hit_mp_miss || p2_hit_buffer || p2_hit_piq || p2_check_in_mshr || io.fencei 446 p2_ready := p2_valid && toPIQEnqReq.ready || !p2_valid 447 p2_fire := toPIQEnqReq.fire() 448 p2_discard := p2_valid && p2_req_cancel 449 450 when(p2_discard) { 451 XSDebug(p"IPrefetchPipe: discard req in s2. vaddr:${p2_vaddr}") 452 } 453 454 when(p2_fire) { 455 XSDebug(p"IPrefetchPipe: send req to PIQ. vaddr:${p2_vaddr} paddr:${p2_paddr}") 456 } 457 458 /** PerfAccumulate */ 459 // the number of prefetch request received from ftq 460 XSPerfAccumulate("prefetch_req_receive", fromFtq.req.fire()) 461 // the number of prefetch request sent to PIQ 462 XSPerfAccumulate("prefetch_req_send", toPIQEnqReq.fire()) 463 /** 464 * Count the number of requests that are filtered for various reasons. 465 * The number of prefetch discard in Performance Accumulator may be 466 * a littel larger the number of really discarded. Because there can 467 * be multiple reasons for a canceled request at the same time. 468 */ 469 // discard prefetch request by fencei instruction 470 XSPerfAccumulate("fdip_prefetch_discard_by_fencei", (p0_discard || p1_discard || p2_discard) && io.fencei) 471 // discard prefetch request by hit hit MSHR 472 XSPerfAccumulate("fdip_prefetch_discard_by_mshr", (p1_discard && p1_check_in_mshr) || (p2_discard && p2_check_in_mshr)) 473 // discard prefetch request by the request is same as pipeline behind 474 XSPerfAccumulate("fdip_prefetch_discard_by_behind", p0_discard && p0_hit_behind) 475 // discard prefetch request by tlb miss 476 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_miss", p1_discard && p1_tlb_resp_miss) 477 // discard prefetch request by tlb except 478 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_except", p1_discard && p1_has_except) 479 // discard prefetch request by hit icache 480 XSPerfAccumulate("fdip_prefetch_discard_by_hit_cache", p2_discard && p2_tag_match) 481 // discard prefetch request by pmp except or mmio 482 XSPerfAccumulate("fdip_prefetch_discard_by_pmp", p2_discard && p2_exception) 483 // discard prefetch request by req is handling by icache mainpipe 484 XSPerfAccumulate("fdip_prefetch_discard_by_hit_mp", p2_discard && p2_hit_mp_miss) 485 // discard prefetch request by hit in prefetch buffer 486 XSPerfAccumulate("fdip_prefetch_discard_by_hit_pf", p2_discard && p2_hit_buffer) 487 // discard prefetch request by hit in prefetch queue 488 XSPerfAccumulate("fdip_prefetch_discard_by_hit_piq", p2_discard && p2_hit_piq) 489} 490 491// class PIQEntry(edge: TLEdgeOut, id: Int)(implicit p: Parameters) extends IPrefetchModule 492// { 493// val io = IO(new Bundle{ 494// val id = Input(UInt((log2Ceil(nPrefetchEntries + PortNumber)).W)) 495 496// val req = Flipped(DecoupledIO(new PIQReq)) 497 498// val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 499// val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 500 501// // write back to Prefetch Buffer 502// val piq_write_ipbuffer = DecoupledIO(new IPFBufferWrite) 503 504// val fencei = Input(Bool()) 505 506// val prefetch_entry_data = DecoupledIO(new PIQData) 507 508// val ongoing_req = ValidIO(UInt(PAddrBits.W)) 509// }) 510 511// val s_idle :: s_memReadReq :: s_memReadResp :: s_write_back :: s_finish:: Nil = Enum(5) 512// val state = RegInit(s_idle) 513 514// //req register 515// val req = Reg(new PIQReq) 516// val req_idx = req.vSetIdx //virtual index 517// val req_tag = get_phy_tag(req.paddr) //physical tag 518 519// val (_, _, refill_done, refill_address_inc) = edge.addr_inc(io.mem_grant) 520 521// //8 for 64 bits bus and 2 for 256 bits 522// val readBeatCnt = Reg(UInt(log2Up(refillCycles).W)) 523// val respDataReg = Reg(Vec(refillCycles,UInt(beatBits.W))) 524 525// //to main pipe s1 526// io.prefetch_entry_data.valid := state =/= s_idle 527// io.prefetch_entry_data.bits.vSetIdx := req_idx 528// io.prefetch_entry_data.bits.ptage := req_tag 529// io.prefetch_entry_data.bits.cacheline := respDataReg.asUInt 530// io.prefetch_entry_data.bits.writeBack := state === s_write_back 531 532// //initial 533// io.mem_acquire.bits := DontCare 534// io.mem_grant.ready := true.B 535// io.piq_write_ipbuffer.bits:= DontCare 536 537// io.req.ready := state === s_idle 538// io.mem_acquire.valid := state === s_memReadReq 539 540// val needflush_r = RegInit(false.B) 541// when (state === s_idle) { needflush_r := false.B } 542// when (state =/= s_idle && io.fencei) { needflush_r := true.B } 543// val needflush = needflush_r | io.fencei 544 545// //state change 546// switch(state){ 547// is(s_idle){ 548// when(io.req.fire()){ 549// readBeatCnt := 0.U 550// state := s_memReadReq 551// req := io.req.bits 552// } 553// } 554 555// // memory request 556// is(s_memReadReq){ 557// when(io.mem_acquire.fire()){ 558// state := s_memReadResp 559// } 560// } 561 562// is(s_memReadResp){ 563// when (edge.hasData(io.mem_grant.bits)) { 564// when (io.mem_grant.fire()) { 565// readBeatCnt := readBeatCnt + 1.U 566// respDataReg(readBeatCnt) := io.mem_grant.bits.data 567// when (readBeatCnt === (refillCycles - 1).U) { 568// assert(refill_done, "refill not done!") 569// state := s_write_back 570// } 571// } 572// } 573// } 574 575// is(s_write_back){ 576// state := Mux(io.piq_write_ipbuffer.fire() || needflush, s_finish, s_write_back) 577// } 578 579// is(s_finish){ 580// state := s_idle 581// } 582// } 583 584// //refill write and meta write 585// //WARNING: Maybe could not finish refill in 1 cycle 586// io.piq_write_ipbuffer.valid := (state === s_write_back) && !needflush 587// io.piq_write_ipbuffer.bits.meta.tag := req_tag 588// io.piq_write_ipbuffer.bits.meta.index := req_idx 589// io.piq_write_ipbuffer.bits.meta.paddr := req.paddr 590// io.piq_write_ipbuffer.bits.data := respDataReg.asUInt 591// io.piq_write_ipbuffer.bits.buffIdx := io.id - PortNumber.U 592 593// io.ongoing_req.valid := state =/= s_idle 594// io.ongoing_req.bits := addrAlign(req.paddr, blockBytes, PAddrBits) 595 596// XSPerfAccumulate("PrefetchEntryReq" + Integer.toString(id, 10), io.req.fire()) 597 598// //mem request 599// io.mem_acquire.bits := edge.Get( 600// fromSource = io.id, 601// toAddress = Cat(req.paddr(PAddrBits - 1, log2Ceil(blockBytes)), 0.U(log2Ceil(blockBytes).W)), 602// lgSize = (log2Up(cacheParams.blockBytes)).U)._2 603// io.mem_acquire.bits.user.lift(PreferCacheKey).foreach(_ := true.B) 604// io.mem_acquire.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.L1InstPrefetch.id.U) 605// } 606 607class PrefetchQueueIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 608 val hartId = Input(UInt(8.W)) 609 val prefetchReq = Flipped(DecoupledIO(new PrefetchReq)) 610 val PIQFilterRead = new PIQFilterRead 611 val PIQRead = new PIQRead 612 val IPFBufferWrite = DecoupledIO(new IPFBufferWrite) 613 /** TileLink Port */ 614 val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 615 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 616 val fencei = Input(Bool()) 617} 618 619class PrefetchQueue(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule { 620 val io = IO(new PrefetchQueueIO(edge)) 621 622 val enqueReq = io.prefetchReq 623 val fromIPrefetch = io.PIQFilterRead.req 624 val toIPrefetch = io.PIQFilterRead.resp 625 val fromMainPipe = io.PIQRead.req 626 val toMainPipe = io.PIQRead.resp 627 val toIPF = io.IPFBufferWrite 628 629 /** Define entryt content and initial FIFO queue */ 630 class PrefetchEntry(implicit p: Parameters) extends IPrefetchBundle { 631 val valid = Bool() 632 val paddr = UInt(PAddrBits.W) 633 val vSetIdx = UInt(idxBits.W) 634 } 635 val PIQ = RegInit(VecInit(Seq.fill(nPrefetchEntries)(0.U.asTypeOf((new PrefetchEntry).cloneType)))) 636 637 class HandleEntry(implicit p: Parameters) extends IPrefetchBundle { 638 val valid = Bool() 639 val paddr = UInt(PAddrBits.W) 640 val vSetIdx = UInt(idxBits.W) 641 val issue = Bool() 642 val finish = Bool() 643 val flash = Bool() 644 val readBeatCnt = UInt(log2Up(refillCycles).W) 645 val respData = Vec(refillCycles, UInt(beatBits.W)) 646 647 def cacheline = respData.asUInt 648 } 649 val handleEntry = RegInit(0.U.asTypeOf((new HandleEntry).cloneType)) 650 651 /** 652 ****************************************************************************** 653 * handle look-up request from IPrefetchPipe 654 * - 1. Receive request from IPrefetch 655 * - 2. Look up whether hit in PIQ and handleEntry 656 * - 3. Send response to IPrefetch at the same cycle 657 ****************************************************************************** 658 */ 659 val fr_block = get_block(fromIPrefetch.paddr) 660 val fr_hit_oh = PIQ.map(e => e.valid && (get_block(e.paddr) === fr_block)) :+ 661 (handleEntry.valid && (get_block(handleEntry.paddr) === fr_block)) 662 val fr_hit = ParallelOR(fr_hit_oh) 663 toIPrefetch.piq_hit := fr_hit 664 // only hit one entry in PIQ 665 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 PIQ for filter") 666 667 668 /** 669 ****************************************************************************** 670 * handle read request from ICacheMainPipe 671 * - 1. Receive request from ICacheMainPipe 672 * - 2. Look up PIQ and handleEntry 673 * - 3. Send response to ICacheMainPipe at the same cycle 674 ****************************************************************************** 675 */ 676 fromMainPipe.foreach(_.ready := DontCare) 677 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 678 /** When hit in PIQ, invalid hitted entry */ 679 val r_hit_oh = r_blocks.map (block => PIQ.map(e => e.valid && (get_block(e.paddr) === block))) 680 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 681 val r_hit_PIQ = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 682 // only hit one entry in PIQ 683 (0 until PortNumber).map (i => 684 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 PIQ") 685 ) 686 687 /** When hit in handleEntry has't been issued, cancel it and return not hit at the same cycle. 688 * When hit in handleEntry has been issued return hit at the same cycle, then return data when transiton finish. 689 * ICacheMainPipe should be blocked during handle handleEntry. 690 */ 691 val r_hit_handle = r_blocks.map(block => handleEntry.valid && (block === get_block(handleEntry.paddr))) 692 val cancelHandleEntry = (((0 until PortNumber).map(i=> fromMainPipe(i).valid && r_hit_handle(i))).reduce(_||_) || io.fencei) && !handleEntry.issue 693 val piq_hit = (0 until PortNumber).map(i => r_hit_handle(i) && handleEntry.issue && !handleEntry.flash && !io.fencei) 694 (0 until PortNumber).foreach (i => { 695 toMainPipe(i).piq_hit := piq_hit(i) 696 toMainPipe(i).cacheline := handleEntry.cacheline 697 toMainPipe(i).data_valid := handleEntry.valid && handleEntry.finish 698 }) 699 val piq_hit_req = RegEnable(piq_hit.reduce(_||_), fromMainPipe.map(_.valid).reduce(_||_)) 700 701 /** 702 ****************************************************************************** 703 * Manage prefetch requests by queue 704 * - 1. Shift: entry shift when there are free entries behind 705 * - 2. Enqueue: receive prefetch request when first entry free 706 * - 3. Free: invalid the entry hitted mainpipe look up 707 * - 4. Fencei PIQ: invalid all PIQ entry when fencei 708 * - 5. Dequeue: move prefetch request from last entry to handle entry 709 * - 6. Cancel handleEntry hitted mainpipe if hasn't been issued 710 * - 7. Valid flash of handleEntry when fencei if has been issued 711 ****************************************************************************** 712 */ 713 /** 1. Shift: entry shift when there are free entries behind */ 714 val move = Wire(Vec(nPrefetchEntries, Bool())) 715 (0 until nPrefetchEntries).map (i => 716 // shift when there are invalid (free) entry behind 717 move(i) := ParallelOR((i+1 until nPrefetchEntries).map(!PIQ(_).valid) :+ !handleEntry.valid) 718 ) 719 (1 until nPrefetchEntries).map (i => 720 when(move(i-1)) { 721 PIQ(i) := PIQ(i-1) 722 } 723 ) 724 725 /** 2. Enqueue: receive prefetch request to PIQ(0) when move(0) otherwise blocked*/ 726 enqueReq.ready := move(0) 727 when(move(0)) { 728 when(enqueReq.valid) { 729 PIQ(0).valid := true.B 730 PIQ(0).vSetIdx := enqueReq.bits.vSetIdx 731 PIQ(0).paddr := enqueReq.bits.paddr 732 }.otherwise { 733 PIQ(0).valid := false.B 734 } 735 } 736 737 /** 3. Free: invalid the entry hitted mainpipe */ 738 (0 until PortNumber) .map (i => 739 when(r_hit_PIQ(i)) { 740 when(move(curr_hit_ptr(i)) && (curr_hit_ptr(i) =/= (nPrefetchEntries-1).U)) { 741 // the hitted entry will be shifted, so invalid next entry 742 PIQ(curr_hit_ptr(i) + 1.U).valid := false.B 743 }.otherwise { 744 PIQ(curr_hit_ptr(i)).valid := false.B 745 } 746 } 747 ) 748 749 /** 4. Fencei: invalid all PIQ entry when fencei */ 750 when(io.fencei) { 751 PIQ.map ( 752 _.valid := false.B 753 ) 754 } 755 756 /** 5. Dequeue: move PIQ.last to handle entry when handle entry is free */ 757 // cancel entry when PIQ.last is hitted by mainpipe 758 val cancel = (0 until PortNumber).map (i => r_hit_PIQ(i) && (curr_hit_ptr(i) === (nPrefetchEntries-1).U)).reduce(_||_) 759 when(move(nPrefetchEntries - 1) && !cancel) { 760 handleEntry.valid := PIQ(nPrefetchEntries - 1).valid 761 handleEntry.paddr := PIQ(nPrefetchEntries - 1).paddr 762 handleEntry.vSetIdx := PIQ(nPrefetchEntries - 1).vSetIdx 763 handleEntry.issue := false.B 764 handleEntry.finish := false.B 765 handleEntry.flash := false.B 766 handleEntry.readBeatCnt := 0.U 767 } 768 769 /** 6. Cancel handleEntry hitted mainpipe or receive fencei if hasn't been issued */ 770 when(cancelHandleEntry) { 771 handleEntry.valid := false.B 772 } 773 774 /** 7. Valid flash of handleEntry when fencei. Drop data from TileLink port */ 775 when(io.fencei) { 776 handleEntry.flash := true.B 777 } 778 779 780 /** 781 ****************************************************************************** 782 * Write Prefetch Data to Prefetch Buffer 783 * - valid signal only hold one cycle 784 ****************************************************************************** 785 */ 786 toIPF.valid := handleEntry.valid && handleEntry.finish 787 toIPF.bits.paddr := handleEntry.paddr 788 toIPF.bits.vSetIdx := handleEntry.vSetIdx 789 toIPF.bits.cacheline := handleEntry.cacheline 790 toIPF.bits.has_hit := piq_hit_req 791 792 793 /** 794 ****************************************************************************** 795 * Tilelink Transition 796 * - 1. Send acquire to L2Cache, after which the request cannot be canceled (issue valid) 797 * - 2. Wait data response from Tilelink 798 * - 3. Get data, valid finish (only one cycle) and invalid issue 799 ****************************************************************************** 800 */ 801 val (_, _, refill_done, refill_address_inc) = edge.addr_inc(io.mem_grant) 802 803 val s_idle :: s_memReadReq :: s_memReadResp :: s_write_back :: Nil = Enum(4) 804 val state = RegInit(s_idle) 805 806 // acquire port 807 io.mem_acquire.bits := edge.Get( 808 fromSource = PortNumber.U, 809 toAddress = Cat(handleEntry.paddr(PAddrBits - 1, log2Ceil(blockBytes)), 0.U(log2Ceil(blockBytes).W)), 810 lgSize = (log2Up(cacheParams.blockBytes)).U)._2 811 io.mem_acquire.bits.user.lift(PreferCacheKey).foreach(_ := true.B) 812 io.mem_acquire.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.L1InstPrefetch.id.U) 813 814 // grant port 815 io.mem_grant.ready := true.B 816 io.mem_acquire.valid := (state === s_memReadReq) 817 818 switch(state) { 819 is(s_idle) { 820 when(handleEntry.valid && !cancelHandleEntry) { 821 handleEntry.issue := true.B 822 state := s_memReadReq 823 } 824 } 825 is(s_memReadReq) { 826 state := Mux(io.mem_acquire.fire(), s_memReadResp, s_memReadReq) 827 } 828 is(s_memReadResp) { 829 when (edge.hasData(io.mem_grant.bits) && io.mem_grant.fire()) { 830 handleEntry.readBeatCnt := handleEntry.readBeatCnt + 1.U 831 handleEntry.respData(handleEntry.readBeatCnt) := io.mem_grant.bits.data 832 when (handleEntry.readBeatCnt === (refillCycles - 1).U) { 833 assert(refill_done, "refill not done!") 834 state := s_write_back 835 when (!io.fencei && !handleEntry.flash) { 836 handleEntry.issue := false.B 837 handleEntry.finish := true.B 838 } 839 } 840 } 841 } 842 is(s_write_back) { 843 state := s_idle 844 handleEntry.valid := false.B 845 } 846 } 847 848 if (env.EnableDifftest) { 849 val diffipfrefill = DifftestModule(new DiffRefillEvent) 850 diffipfrefill.clock := clock 851 diffipfrefill.coreid := io.hartId 852 diffipfrefill.index := 3.U 853 diffipfrefill.valid := handleEntry.valid && handleEntry.finish 854 diffipfrefill.addr := handleEntry.paddr 855 diffipfrefill.data := handleEntry.cacheline.asTypeOf(diffipfrefill.data) 856 } 857} 858 859class FDIPPrefetchIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 860 /** commen */ 861 val fencei = Input(Bool()) 862 val hartId = Input(UInt(8.W)) 863 864 /** Prefetch Mainpipe IO */ 865 val ftqReq = Flipped(new FtqPrefechBundle) 866 val iTLBInter = new TlbRequestIO 867 val pmp = new ICachePMPBundle 868 val metaReadReq = Decoupled(new ICacheMetaReadReqBundle) 869 val metaReadResp = Input(new ICacheMetaReadRespBundle) 870 val mshrInfo = Flipped(Vec(PortNumber, ValidIO(UInt(PAddrBits.W)))) 871 val mainPipeMissInfo = Flipped(Vec(PortNumber,ValidIO(new MainPipeMissInfo))) 872 873 /** Prefetch Buffer IO */ 874 val IPFBufferRead = new IPFBufferRead 875 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 876 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 877 val IPFReplacer = new IPFReplacer 878 879 /** Prefetch Queue IO */ 880 val PIQRead = new PIQRead 881 val mem_acquire = new DecoupledIO(new TLBundleA(edge.bundle)) 882 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 883} 884 885class FDIPPrefetch(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule 886{ 887 val io = IO(new FDIPPrefetchIO(edge)) 888 889 val prefetchPipe = Module(new IPrefetchPipe) 890 val prefetchQueue = Module(new PrefetchQueue(edge)) 891 val prefetchBuffer = Module(new PrefetchBuffer) 892 893 io.metaReadReq <> prefetchPipe.io.metaReadReq 894 io.metaWrite <> prefetchBuffer.io.metaWrite 895 io.dataWrite <> prefetchBuffer.io.dataWrite 896 io.IPFReplacer <> prefetchBuffer.io.IPFReplacer 897 io.mem_acquire <> prefetchQueue.io.mem_acquire 898 io.mem_grant <> prefetchQueue.io.mem_grant 899 900 // outside 901 prefetchPipe.io.fencei <> io.fencei 902 prefetchPipe.io.ftqReq <> io.ftqReq 903 prefetchPipe.io.iTLBInter <> io.iTLBInter 904 prefetchPipe.io.pmp <> io.pmp 905 prefetchPipe.io.metaReadResp <> io.metaReadResp 906 prefetchPipe.io.mshrInfo <> io.mshrInfo 907 prefetchPipe.io.mainPipeMissInfo <> io.mainPipeMissInfo 908 909 // outside 910 prefetchBuffer.io.hartId <> io.hartId 911 prefetchBuffer.io.fencei <> io.fencei 912 prefetchBuffer.io.IPFBufferRead <> io.IPFBufferRead 913 // inside 914 prefetchBuffer.io.IPFFilterRead <> prefetchPipe.io.IPFFilterRead 915 prefetchBuffer.io.IPFBufferWrite <> prefetchQueue.io.IPFBufferWrite 916 917 // outside 918 prefetchQueue.io.hartId <> io.hartId 919 prefetchQueue.io.fencei <> io.fencei 920 prefetchQueue.io.PIQRead <> io.PIQRead 921 // inside 922 prefetchQueue.io.prefetchReq <> prefetchPipe.io.prefetchReq 923 prefetchQueue.io.PIQFilterRead <> prefetchPipe.io.PIQFilterRead 924} 925