1/*************************************************************************************** 2 * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3 * Copyright (c) 2020-2021 Peng Cheng Laboratory 4 * 5 * XiangShan is licensed under Mulan PSL v2. 6 * You can use this software according to the terms and conditions of the Mulan PSL v2. 7 * You may obtain a copy of Mulan PSL v2 at: 8 * http://license.coscl.org.cn/MulanPSL2 9 * 10 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13 * 14 * See the Mulan PSL v2 for more details. 15 ***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import difftest._ 23import freechips.rocketchip.tilelink._ 24import utils._ 25import xiangshan.cache.mmu._ 26import xiangshan.frontend._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28import huancun.PreferCacheKey 29import xiangshan.XSCoreParamsKey 30import utility._ 31 32 33abstract class IPrefetchBundle(implicit p: Parameters) extends ICacheBundle 34abstract class IPrefetchModule(implicit p: Parameters) extends ICacheModule 35 36//TODO: remove this 37object DebugFlags { 38 val fdip = false 39} 40 41class PIQData(implicit p: Parameters) extends IPrefetchBundle { 42 val ptage = UInt(tagBits.W) 43 val vSetIdx = UInt(idxBits.W) 44 val cacheline = UInt(blockBits.W) 45 val writeBack = Bool() 46} 47 48class PIQToMainPipe(implicit p: Parameters) extends IPrefetchBundle{ 49 val info = DecoupledIO(new PIQData) 50} 51/* need change name */ 52class MainPipeMissInfo(implicit p: Parameters) extends IPrefetchBundle { 53 val ptage = UInt(tagBits.W) 54 val vSetIdx = UInt(idxBits.W) 55} 56 57class PrefetchReq(implicit p: Parameters) extends IPrefetchBundle{ 58 val paddr = UInt(PAddrBits.W) 59 val vSetIdx = UInt(idxBits.W) 60} 61 62class PrefetchBufferIO(implicit p: Parameters) extends IPrefetchBundle { 63 val hartId = Input(UInt(8.W)) 64 val fencei = Input(Bool()) 65 val IPFFilterRead = new IPFBufferFilterRead 66 val IPFBufferRead = new IPFBufferRead 67 val IPFBufferWrite = Flipped(DecoupledIO(new IPFBufferWrite)) 68 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 69 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 70 val IPFReplacer = new IPFReplacer 71} 72 73class PrefetchBuffer(implicit p: Parameters) extends IPrefetchModule 74{ 75 val io = IO(new PrefetchBufferIO) 76 77 val fromIPrefetch = io.IPFFilterRead.req 78 val toIPrefetch = io.IPFFilterRead.resp 79 val fromMainPipe = io.IPFBufferRead.req 80 val toMainPipe = io.IPFBufferRead.resp 81 val fromPIQ = io.IPFBufferWrite 82 val toICacheMeta = io.metaWrite 83 val toICacheData = io.dataWrite 84 val fromReplacer = io.IPFReplacer 85 86 class IPFBufferEntryMeta(implicit p: Parameters) extends IPrefetchBundle 87 { 88 val valid = Bool() 89 val paddr = UInt(PAddrBits.W) 90 val vSetIdx = UInt(idxBits.W) 91 val confidence = UInt(log2Ceil(maxIPFMoveConf + 1).W) 92 } 93 94 class IPFBufferEntryData(implicit p: Parameters) extends IPrefetchBundle 95 { 96 val cacheline = UInt(blockBits.W) 97 } 98 99 def InitQueue[T <: Data](entry: T, size: Int): Vec[T] ={ 100 return RegInit(VecInit(Seq.fill(size)(0.U.asTypeOf(entry.cloneType)))) 101 } 102 103 val meta_buffer = InitQueue(new IPFBufferEntryMeta, size = nIPFBufferSize) 104 val data_buffer = InitQueue(new IPFBufferEntryData, size = nIPFBufferSize) 105 106 /** 107 ****************************************************************************** 108 * handle look-up request from IPrefetchPipe 109 * - 1. Receive request from IPrefetch 110 * - 2. Look up whether hit in meta_buffer 111 * - 3. Send response to IPrefetch at the same cycle 112 ****************************************************************************** 113 */ 114 val fr_block = get_block(fromIPrefetch.paddr) 115 val fr_hit_oh = meta_buffer.map(e => e.valid && (get_block(e.paddr) === fr_block)) 116 toIPrefetch.ipf_hit := ParallelOR(fr_hit_oh) 117 // only hit one entry in meta_buffer at the same time 118 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 prefetch buffer for filter") 119 120 /** 121 ****************************************************************************** 122 * handle read request from ICacheMainPipe 123 * - 1. Receive request from ICacheMainPipe 124 * - 2. Look up whether hit in meta_buffer and get hitted data 125 * - 3. Send response to ICacheMainPipe at the same cycle 126 ****************************************************************************** 127 */ 128 fromMainPipe.foreach(_.ready := DontCare) 129 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 130 val r_hit_oh = r_blocks.map (ptag => meta_buffer.map(e => e.valid && (get_block(e.paddr) === ptag))) 131 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 132 val r_hit_data = r_hit_oh.map (oh => Mux1H(oh, data_buffer.map(_.cacheline))) 133 val r_hit = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 134 (0 until PortNumber).foreach (i => { 135 toMainPipe(i).ipf_hit := r_hit(i) 136 toMainPipe(i).cacheline := r_hit_data(i) 137 // only hit one entry in meta_buffer 138 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 prefetch buffer") 139 XSPerfAccumulate("fdip_mainpipe_hit_in_ipf_" + i, r_hit(i)) 140 }) 141 142 143 /** 144 ****************************************************************************** 145 * move the entry arriving max confidence to icache 146 * - 1. Chose a moved entry from the entries that reach max confidence (LSB first) 147 * - 2. Get victim way of array from replacer of ICacheMainPipe 148 * - 3. Send write req to metaArray and dataArray and ensure fire together 149 ****************************************************************************** 150 */ 151 val move_oh = meta_buffer.map (e => e.confidence === maxIPFMoveConf.U && e.valid) 152 val move_need = ParallelOR(move_oh) 153 val curr_move_ptr = PriorityEncoder(move_oh) 154 155 fromReplacer.vsetIdx := meta_buffer(curr_move_ptr).vSetIdx 156 157 toICacheMeta.valid := move_need 158 toICacheMeta.bits.generate( 159 tag = get_phy_tag(meta_buffer(curr_move_ptr).paddr), 160 idx = meta_buffer(curr_move_ptr).vSetIdx, 161 waymask = fromReplacer.waymask, 162 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0)) 163 164 toICacheData.valid := move_need 165 toICacheData.bits.generate( 166 data = data_buffer(curr_move_ptr).cacheline, 167 idx = meta_buffer(curr_move_ptr).vSetIdx, 168 waymask = fromReplacer.waymask, 169 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0), 170 paddr = meta_buffer(curr_move_ptr).paddr) 171 172 // TODO: how to ensure 173 assert((toICacheMeta.fire && toICacheData.fire) || (!toICacheMeta.fire && !toICacheData.fire), 174 "meta and data array need fire at same time") 175 176 XSPerfAccumulate("fdip_move_to_icache", toICacheMeta.fire && toICacheData.fire) 177 178 when(toICacheMeta.fire) { 179 XSDebug(p"PrefetchBuffer: write prefetch data to icache . vSetIdx:${meta_buffer(curr_move_ptr).vSetIdx} " + 180 p"paddr:${meta_buffer(curr_move_ptr).paddr}") 181 } 182 183 /** 184 ****************************************************************************** 185 * Prefetch buffer write logic 186 * Read old data when read and write occur in the same cycle 187 * There are 4 situations that can cause meta_buffer and data_buffer to be written 188 * - 1. ICacheMainPipe read hit that need to change some status 189 * - 2. Move prefetch cacheline to icache 190 * - 3. Receive write request from ICacheMissUnit 191 * - 4. Receive fencei req that need to flush all buffer 192 * Priority: 4 > 3 > 2 > 1 193 * The order of the code determines the priority, with the following priority being higher 194 ****************************************************************************** 195 */ 196 197 /** 1. confidence++ for every ICacheMainPipe hit */ 198 (0 until PortNumber) .map (i => 199 when(r_hit(i)) { 200 // handle overflow 201 when(meta_buffer(curr_hit_ptr(i)).confidence =/= maxIPFMoveConf.U) { 202 meta_buffer(curr_hit_ptr(i)).confidence := meta_buffer(curr_hit_ptr(i)).confidence + 1.U 203 } 204 } 205 ) 206 207 /** 2. set entry invalid after move to icache */ 208 when(toICacheMeta.fire) { 209 meta_buffer(curr_move_ptr).valid := false.B 210 } 211 212 /** 3. write prefetch data to entry */ 213 // try to get a free entry ptr 214 val free_oh = meta_buffer.map (_.valid === false.B) 215 val has_free = ParallelOR(free_oh) 216 val ptr_from_free = PriorityEncoder(free_oh) 217 // get a ptr from random replacement policy 218 val replacer = ReplacementPolicy.fromString(Some("random"), nIPFBufferSize) 219 // write to free entry if has, otherwise random entry 220 val curr_write_ptr = Mux(has_free, ptr_from_free, replacer.way) 221 fromPIQ.ready := DontCare 222 when(fromPIQ.valid) { 223 meta_buffer(curr_write_ptr).valid := true.B 224 meta_buffer(curr_write_ptr).confidence := fromPIQ.bits.has_hit.asUInt 225 meta_buffer(curr_write_ptr).paddr := fromPIQ.bits.paddr 226 meta_buffer(curr_write_ptr).vSetIdx := fromPIQ.bits.vSetIdx 227 data_buffer(curr_write_ptr).cacheline := fromPIQ.bits.cacheline 228 } 229 230 if(DebugFlags.fdip) { 231 when(fromPIQ.valid) { 232 printf(p"PrefetchBuffer: receive prefetch data. vSetIdx:${fromPIQ.bits.vSetIdx} paddr:${fromPIQ.bits.paddr}") 233 } 234 } 235 236 XSPerfAccumulate("fdip_vitcm_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence =/= 0.U)) 237 XSPerfAccumulate("fdip_vitcm_no_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence === 0.U)) 238 239 /** 4. flush all prefetch buffer when fencei */ 240 when(io.fencei) { 241 meta_buffer.foreach { b => 242 b.valid := false.B 243 b.confidence := 0.U 244 } 245 } 246 247 XSPerfAccumulate("fdip_fencei_cycle", io.fencei) 248 249 if (env.EnableDifftest) { 250 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 251 difftest.coreid := io.hartId 252 difftest.index := 6.U 253 difftest.valid := toICacheData.fire 254 difftest.addr := toICacheData.bits.paddr 255 difftest.data := toICacheData.bits.data.asTypeOf(difftest.data) 256 } 257} 258 259class IPredfetchIO(implicit p: Parameters) extends IPrefetchBundle { 260 val ftqReq = Flipped(new FtqPrefechBundle) 261 val iTLBInter = new TlbRequestIO 262 val pmp = new ICachePMPBundle 263 val metaReadReq = Decoupled(new ICacheMetaReadReqBundle) 264 val metaReadResp = Input(new ICacheMetaReadRespBundle) 265 val prefetchReq = DecoupledIO(new PrefetchReq) 266 val mshrInfo = Flipped(Vec(PortNumber, ValidIO(UInt(PAddrBits.W)))) 267 val IPFFilterRead = Flipped(new IPFBufferFilterRead) 268 val PIQFilterRead = Flipped(new PIQFilterRead) 269 /** icache main pipe to prefetch pipe */ 270 val mainPipeMissInfo = Flipped(Vec(PortNumber,ValidIO(new MainPipeMissInfo))) 271 val fencei = Input(Bool()) 272} 273 274class IPrefetchPipe(implicit p: Parameters) extends IPrefetchModule 275{ 276 val io = IO(new IPredfetchIO) 277 278 val enableBit = RegInit(false.B) 279 enableBit := false.B 280 281 val fromFtq = io.ftqReq 282 val mainPipeMissInfo = io.mainPipeMissInfo 283 val (toITLB, fromITLB) = (io.iTLBInter.req, io.iTLBInter.resp) 284 val (toIMeta, fromIMeta, fromIMetaValid) = (io.metaReadReq, io.metaReadResp.metaData, io.metaReadResp.entryValid) 285 val (toIPFBuffer, fromIPFBuffer) = (io.IPFFilterRead.req, io.IPFFilterRead.resp) 286 val (toPIQ, fromPIQ) = (io.PIQFilterRead.req, io.PIQFilterRead.resp) 287 val (toPMP, fromPMP) = (io.pmp.req, io.pmp.resp) 288 val fromMSHR = io.mshrInfo 289 val toPIQEnqReq = io.prefetchReq 290 291 val p0_fire, p1_fire, p2_fire = WireInit(false.B) 292 val p0_discard, p1_discard, p2_discard = WireInit(false.B) 293 val p0_ready, p1_ready, p2_ready = WireInit(false.B) 294 295 /** 296 ****************************************************************************** 297 * IPrefetch Stage 0 298 * - 1. send req to IMeta 299 * - 2. send req to ITLB (no blocked) 300 * - 3. check last req 301 ****************************************************************************** 302 */ 303 val p0_valid = fromFtq.req.valid 304 305 val p0_vaddr = addrAlign(fromFtq.req.bits.target, blockBytes, VAddrBits) 306 val p0_vaddr_reg = RegEnable(p0_vaddr, p0_fire) 307 val p0_req_cancel = Wire(Bool()) 308 309 /** 1. send req to IMeta */ 310 toIMeta.valid := p0_valid && !p0_req_cancel 311 toIMeta.bits.idx := get_idx(p0_vaddr) 312 313 /** 2. send req to ITLB (no blocked) */ 314 toITLB.valid := p0_valid && !p0_req_cancel 315 toITLB.bits.size := 3.U // TODO: fix the size 316 toITLB.bits.vaddr := p0_vaddr 317 toITLB.bits.debug.pc := p0_vaddr 318 toITLB.bits.kill := DontCare 319 toITLB.bits.cmd := TlbCmd.exec 320 toITLB.bits.debug.robIdx := DontCare 321 toITLB.bits.debug.isFirstIssue := DontCare 322 toITLB.bits.memidx := DontCare 323 toITLB.bits.no_translate := false.B 324 fromITLB.ready := true.B 325 // TODO: whether to handle tlb miss for prefetch 326 io.iTLBInter.req_kill := false.B 327 328 /** FTQ request port */ 329 fromFtq.req.ready := p0_ready 330 331 /** 3. Check last req: request from FTQ is same as last time or behind */ 332 val p0_hit_behind = Wire(Bool()) 333 334 /** stage 0 control */ 335 // Cancel request when prefetch not enable or the request from FTQ is same as last time 336 p0_req_cancel := !enableBit || p0_hit_behind || io.fencei 337 p0_ready := p0_req_cancel || p1_ready && toITLB.ready && toIMeta.ready 338 p0_fire := p0_valid && p1_ready && toITLB.ready && toIMeta.ready && enableBit && !p0_req_cancel 339 p0_discard := p0_valid && p0_req_cancel 340 341 if(DebugFlags.fdip) { 342 when(p0_discard) { 343 printf(p"IPrefetchPipe: discard req in s0.") 344 } 345 } 346 347 /** 348 ****************************************************************************** 349 * IPrefetch Stage 1 350 * - 1. Receive resp from ITLB (no blocked) 351 * - 2. Receive resp from IMeta 352 * - 3. Check MSHR 353 ****************************************************************************** 354 */ 355 val p1_valid = generatePipeControl(lastFire = p0_fire, thisFire = p1_fire || p1_discard, thisFlush = false.B, lastFlush = false.B) 356 357 val p1_vaddr = RegEnable(p0_vaddr, p0_fire) 358 val p1_req_cancel = Wire(Bool()) 359 360 /** 1. Receive resp from ITLB (no blocked) */ 361 val p1_tlb_resp_miss = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.miss) 362 val p1_tlb_resp_paddr = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.paddr(0)) 363 val p1_tlb_resp_pf = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).pf.instr) 364 val p1_tlb_resp_af = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).af.instr) 365 val p1_exception = VecInit(Seq(p1_tlb_resp_pf, p1_tlb_resp_af)) 366 val p1_has_except = p1_exception.reduce(_ || _) 367 val p1_paddr = p1_tlb_resp_paddr 368 369 /** 2. Receive resp from IMeta. Register the data first because of strict timing. */ 370 val p1_meta_ptags_reg = RegEnable(VecInit(fromIMeta.map(way => way.tag)), RegNext(p0_fire)) 371 val p1_meta_valids_reg = RegEnable(fromIMetaValid, RegNext(p0_fire)) 372 373 /** 3. Check MSHR */ 374 val p1_check_in_mshr = VecInit(fromMSHR.map(mshr => mshr.valid && mshr.bits === addrAlign(p1_paddr, blockBytes, PAddrBits))).reduce(_||_) 375 376 /** Stage 1 control */ 377 p1_req_cancel := p1_check_in_mshr || p1_tlb_resp_miss || p1_has_except || io.fencei 378 p1_ready := p1_valid && p2_ready || !p1_valid 379 p1_fire := p1_valid && !p1_req_cancel && p2_ready && enableBit 380 p1_discard := p1_valid && p1_req_cancel 381 382 when(p1_discard) { 383 XSDebug(p"IPrefetchPipe: discard req in s1. vaddr:${p1_vaddr}") 384 } 385 386 /** 387 ****************************************************************************** 388 * IPrefetch Stage 2 389 * - 1. IMeta Check: check whether req hit in icache 390 * - 2. PMP Check: send req and receive resq in the same cycle 391 * - 3. Check miss handling by main pipe 392 * - 4. Prefetch buffer Check 393 * - 5. Prefetch Queue Check 394 * - 6. MSHR check 395 * - 7. send req to piq 396 ****************************************************************************** 397 */ 398 val p2_valid = generatePipeControl(lastFire = p1_fire, thisFire = p2_fire || p2_discard, thisFlush = false.B, lastFlush = false.B) 399 400 val p2_paddr = RegEnable(p1_paddr, p1_fire) 401 val p2_vaddr = RegEnable(p1_vaddr, p1_fire) 402 val p2_req_cancel = Wire(Bool()) 403 val p2_vidx = get_idx(p2_vaddr) 404 405 p0_hit_behind := (p0_vaddr === p1_vaddr) || (p0_vaddr === p2_vaddr) 406 407 /** 1. IMeta Check */ 408 val p2_ptag = get_phy_tag(p2_paddr) 409 val p2_tag_eq_vec = VecInit(p1_meta_ptags_reg.map(_ === p2_ptag )) 410 val p2_tag_match_vec = VecInit(p2_tag_eq_vec.zipWithIndex.map{ case(way_tag_eq, w) => way_tag_eq && p1_meta_valids_reg(w)}) 411 val p2_tag_match = DataHoldBypass(ParallelOR(p2_tag_match_vec), RegNext(p1_fire)) 412 413 /** 2. PMP Check */ 414 // TODO: When fromPMP.mmio is low but fromPMP.instr is high, which is a except? 415 val p2_exception = DataHoldBypass((fromPMP.mmio && !fromPMP.instr) || fromPMP.instr, RegNext(p1_fire)) 416 // PMP request 417 toPMP.valid := p2_valid 418 toPMP.bits.addr := p2_paddr 419 toPMP.bits.size := 3.U 420 toPMP.bits.cmd := TlbCmd.exec 421 422 /** 3. Check miss handling by main pipe */ 423 val p2_hit_mp_miss = VecInit((0 until PortNumber).map(i => 424 mainPipeMissInfo(i).valid && (mainPipeMissInfo(i).bits.ptage === p2_ptag && 425 (mainPipeMissInfo(i).bits.vSetIdx === p2_vidx)))).reduce(_||_) 426 427 /** 4. Prefetch buffer Check */ 428 toIPFBuffer.paddr := p2_paddr 429 val p2_hit_buffer = fromIPFBuffer.ipf_hit 430 431 /** 5. Prefetch Queue Check */ 432 toPIQ.paddr := p2_paddr 433 val p2_hit_piq = fromPIQ.piq_hit 434 435 /** 6. MSHR check */ 436 val p2_check_in_mshr = VecInit(fromMSHR.map(mshr => mshr.valid && mshr.bits === addrAlign(p2_paddr, blockBytes, PAddrBits))).reduce(_||_) 437 438 /** 7. send req to piq */ 439 toPIQEnqReq.valid := p2_valid && !p2_req_cancel 440 toPIQEnqReq.bits.paddr := p2_paddr 441 toPIQEnqReq.bits.vSetIdx := p2_vidx 442 443 /** Stage 2 control */ 444 p2_req_cancel := p2_tag_match || p2_exception || p2_hit_mp_miss || p2_hit_buffer || p2_hit_piq || p2_check_in_mshr || io.fencei 445 p2_ready := p2_valid && toPIQEnqReq.ready || !p2_valid 446 p2_fire := toPIQEnqReq.fire() 447 p2_discard := p2_valid && p2_req_cancel 448 449 when(p2_discard) { 450 XSDebug(p"IPrefetchPipe: discard req in s2. vaddr:${p2_vaddr}") 451 } 452 453 when(p2_fire) { 454 XSDebug(p"IPrefetchPipe: send req to PIQ. vaddr:${p2_vaddr} paddr:${p2_paddr}") 455 } 456 457 /** PerfAccumulate */ 458 // the number of prefetch request received from ftq 459 XSPerfAccumulate("prefetch_req_receive", fromFtq.req.fire()) 460 // the number of prefetch request sent to PIQ 461 XSPerfAccumulate("prefetch_req_send", toPIQEnqReq.fire()) 462 /** 463 * Count the number of requests that are filtered for various reasons. 464 * The number of prefetch discard in Performance Accumulator may be 465 * a littel larger the number of really discarded. Because there can 466 * be multiple reasons for a canceled request at the same time. 467 */ 468 // discard prefetch request by fencei instruction 469 XSPerfAccumulate("fdip_prefetch_discard_by_fencei", (p0_discard || p1_discard || p2_discard) && io.fencei) 470 // discard prefetch request by hit hit MSHR 471 XSPerfAccumulate("fdip_prefetch_discard_by_mshr", (p1_discard && p1_check_in_mshr) || (p2_discard && p2_check_in_mshr)) 472 // discard prefetch request by the request is same as pipeline behind 473 XSPerfAccumulate("fdip_prefetch_discard_by_behind", p0_discard && p0_hit_behind) 474 // discard prefetch request by tlb miss 475 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_miss", p1_discard && p1_tlb_resp_miss) 476 // discard prefetch request by tlb except 477 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_except", p1_discard && p1_has_except) 478 // discard prefetch request by hit icache 479 XSPerfAccumulate("fdip_prefetch_discard_by_hit_cache", p2_discard && p2_tag_match) 480 // discard prefetch request by pmp except or mmio 481 XSPerfAccumulate("fdip_prefetch_discard_by_pmp", p2_discard && p2_exception) 482 // discard prefetch request by req is handling by icache mainpipe 483 XSPerfAccumulate("fdip_prefetch_discard_by_hit_mp", p2_discard && p2_hit_mp_miss) 484 // discard prefetch request by hit in prefetch buffer 485 XSPerfAccumulate("fdip_prefetch_discard_by_hit_pf", p2_discard && p2_hit_buffer) 486 // discard prefetch request by hit in prefetch queue 487 XSPerfAccumulate("fdip_prefetch_discard_by_hit_piq", p2_discard && p2_hit_piq) 488} 489 490// class PIQEntry(edge: TLEdgeOut, id: Int)(implicit p: Parameters) extends IPrefetchModule 491// { 492// val io = IO(new Bundle{ 493// val id = Input(UInt((log2Ceil(nPrefetchEntries + PortNumber)).W)) 494 495// val req = Flipped(DecoupledIO(new PIQReq)) 496 497// val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 498// val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 499 500// // write back to Prefetch Buffer 501// val piq_write_ipbuffer = DecoupledIO(new IPFBufferWrite) 502 503// val fencei = Input(Bool()) 504 505// val prefetch_entry_data = DecoupledIO(new PIQData) 506 507// val ongoing_req = ValidIO(UInt(PAddrBits.W)) 508// }) 509 510// val s_idle :: s_memReadReq :: s_memReadResp :: s_write_back :: s_finish:: Nil = Enum(5) 511// val state = RegInit(s_idle) 512 513// //req register 514// val req = Reg(new PIQReq) 515// val req_idx = req.vSetIdx //virtual index 516// val req_tag = get_phy_tag(req.paddr) //physical tag 517 518// val (_, _, refill_done, refill_address_inc) = edge.addr_inc(io.mem_grant) 519 520// //8 for 64 bits bus and 2 for 256 bits 521// val readBeatCnt = Reg(UInt(log2Up(refillCycles).W)) 522// val respDataReg = Reg(Vec(refillCycles,UInt(beatBits.W))) 523 524// //to main pipe s1 525// io.prefetch_entry_data.valid := state =/= s_idle 526// io.prefetch_entry_data.bits.vSetIdx := req_idx 527// io.prefetch_entry_data.bits.ptage := req_tag 528// io.prefetch_entry_data.bits.cacheline := respDataReg.asUInt 529// io.prefetch_entry_data.bits.writeBack := state === s_write_back 530 531// //initial 532// io.mem_acquire.bits := DontCare 533// io.mem_grant.ready := true.B 534// io.piq_write_ipbuffer.bits:= DontCare 535 536// io.req.ready := state === s_idle 537// io.mem_acquire.valid := state === s_memReadReq 538 539// val needflush_r = RegInit(false.B) 540// when (state === s_idle) { needflush_r := false.B } 541// when (state =/= s_idle && io.fencei) { needflush_r := true.B } 542// val needflush = needflush_r | io.fencei 543 544// //state change 545// switch(state){ 546// is(s_idle){ 547// when(io.req.fire()){ 548// readBeatCnt := 0.U 549// state := s_memReadReq 550// req := io.req.bits 551// } 552// } 553 554// // memory request 555// is(s_memReadReq){ 556// when(io.mem_acquire.fire()){ 557// state := s_memReadResp 558// } 559// } 560 561// is(s_memReadResp){ 562// when (edge.hasData(io.mem_grant.bits)) { 563// when (io.mem_grant.fire()) { 564// readBeatCnt := readBeatCnt + 1.U 565// respDataReg(readBeatCnt) := io.mem_grant.bits.data 566// when (readBeatCnt === (refillCycles - 1).U) { 567// assert(refill_done, "refill not done!") 568// state := s_write_back 569// } 570// } 571// } 572// } 573 574// is(s_write_back){ 575// state := Mux(io.piq_write_ipbuffer.fire() || needflush, s_finish, s_write_back) 576// } 577 578// is(s_finish){ 579// state := s_idle 580// } 581// } 582 583// //refill write and meta write 584// //WARNING: Maybe could not finish refill in 1 cycle 585// io.piq_write_ipbuffer.valid := (state === s_write_back) && !needflush 586// io.piq_write_ipbuffer.bits.meta.tag := req_tag 587// io.piq_write_ipbuffer.bits.meta.index := req_idx 588// io.piq_write_ipbuffer.bits.meta.paddr := req.paddr 589// io.piq_write_ipbuffer.bits.data := respDataReg.asUInt 590// io.piq_write_ipbuffer.bits.buffIdx := io.id - PortNumber.U 591 592// io.ongoing_req.valid := state =/= s_idle 593// io.ongoing_req.bits := addrAlign(req.paddr, blockBytes, PAddrBits) 594 595// XSPerfAccumulate("PrefetchEntryReq" + Integer.toString(id, 10), io.req.fire()) 596 597// //mem request 598// io.mem_acquire.bits := edge.Get( 599// fromSource = io.id, 600// toAddress = Cat(req.paddr(PAddrBits - 1, log2Ceil(blockBytes)), 0.U(log2Ceil(blockBytes).W)), 601// lgSize = (log2Up(cacheParams.blockBytes)).U)._2 602// io.mem_acquire.bits.user.lift(PreferCacheKey).foreach(_ := true.B) 603// io.mem_acquire.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.L1InstPrefetch.id.U) 604// } 605 606class PrefetchQueueIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 607 val hartId = Input(UInt(8.W)) 608 val prefetchReq = Flipped(DecoupledIO(new PrefetchReq)) 609 val PIQFilterRead = new PIQFilterRead 610 val PIQRead = new PIQRead 611 val IPFBufferWrite = DecoupledIO(new IPFBufferWrite) 612 /** TileLink Port */ 613 val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 614 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 615 val fencei = Input(Bool()) 616} 617 618class PrefetchQueue(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule { 619 val io = IO(new PrefetchQueueIO(edge)) 620 621 val enqueReq = io.prefetchReq 622 val fromIPrefetch = io.PIQFilterRead.req 623 val toIPrefetch = io.PIQFilterRead.resp 624 val fromMainPipe = io.PIQRead.req 625 val toMainPipe = io.PIQRead.resp 626 val toIPF = io.IPFBufferWrite 627 628 /** Define entryt content and initial FIFO queue */ 629 class PrefetchEntry(implicit p: Parameters) extends IPrefetchBundle { 630 val valid = Bool() 631 val paddr = UInt(PAddrBits.W) 632 val vSetIdx = UInt(idxBits.W) 633 } 634 val PIQ = RegInit(VecInit(Seq.fill(nPrefetchEntries)(0.U.asTypeOf((new PrefetchEntry).cloneType)))) 635 636 class HandleEntry(implicit p: Parameters) extends IPrefetchBundle { 637 val valid = Bool() 638 val paddr = UInt(PAddrBits.W) 639 val vSetIdx = UInt(idxBits.W) 640 val issue = Bool() 641 val finish = Bool() 642 val flash = Bool() 643 val readBeatCnt = UInt(log2Up(refillCycles).W) 644 val respData = Vec(refillCycles, UInt(beatBits.W)) 645 646 def cacheline = respData.asUInt 647 } 648 val handleEntry = RegInit(0.U.asTypeOf((new HandleEntry).cloneType)) 649 650 /** 651 ****************************************************************************** 652 * handle look-up request from IPrefetchPipe 653 * - 1. Receive request from IPrefetch 654 * - 2. Look up whether hit in PIQ and handleEntry 655 * - 3. Send response to IPrefetch at the same cycle 656 ****************************************************************************** 657 */ 658 val fr_block = get_block(fromIPrefetch.paddr) 659 val fr_hit_oh = PIQ.map(e => e.valid && (get_block(e.paddr) === fr_block)) :+ 660 (handleEntry.valid && (get_block(handleEntry.paddr) === fr_block)) 661 val fr_hit = ParallelOR(fr_hit_oh) 662 toIPrefetch.piq_hit := fr_hit 663 // only hit one entry in PIQ 664 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 PIQ for filter") 665 666 667 /** 668 ****************************************************************************** 669 * handle read request from ICacheMainPipe 670 * - 1. Receive request from ICacheMainPipe 671 * - 2. Look up PIQ and handleEntry 672 * - 3. Send response to ICacheMainPipe at the same cycle 673 ****************************************************************************** 674 */ 675 fromMainPipe.foreach(_.ready := DontCare) 676 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 677 /** When hit in PIQ, invalid hitted entry */ 678 val r_hit_oh = r_blocks.map (block => PIQ.map(e => e.valid && (get_block(e.paddr) === block))) 679 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 680 val r_hit_PIQ = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 681 // only hit one entry in PIQ 682 (0 until PortNumber).map (i => 683 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 PIQ") 684 ) 685 686 /** When hit in handleEntry has't been issued, cancel it and return not hit at the same cycle. 687 * When hit in handleEntry has been issued return hit at the same cycle, then return data when transiton finish. 688 * ICacheMainPipe should be blocked during handle handleEntry. 689 */ 690 val r_hit_handle = r_blocks.map(block => handleEntry.valid && (block === get_block(handleEntry.paddr))) 691 val cancelHandleEntry = (((0 until PortNumber).map(i=> fromMainPipe(i).valid && r_hit_handle(i))).reduce(_||_) || io.fencei) && !handleEntry.issue 692 val piq_hit = (0 until PortNumber).map(i => r_hit_handle(i) && handleEntry.issue && !handleEntry.flash && !io.fencei) 693 (0 until PortNumber).foreach (i => { 694 toMainPipe(i).piq_hit := piq_hit(i) 695 toMainPipe(i).cacheline := handleEntry.cacheline 696 toMainPipe(i).data_valid := handleEntry.valid && handleEntry.finish 697 }) 698 val piq_hit_req = RegEnable(piq_hit.reduce(_||_), fromMainPipe.map(_.valid).reduce(_||_)) 699 700 /** 701 ****************************************************************************** 702 * Manage prefetch requests by queue 703 * - 1. Shift: entry shift when there are free entries behind 704 * - 2. Enqueue: receive prefetch request when first entry free 705 * - 3. Free: invalid the entry hitted mainpipe look up 706 * - 4. Fencei PIQ: invalid all PIQ entry when fencei 707 * - 5. Dequeue: move prefetch request from last entry to handle entry 708 * - 6. Cancel handleEntry hitted mainpipe if hasn't been issued 709 * - 7. Valid flash of handleEntry when fencei if has been issued 710 ****************************************************************************** 711 */ 712 /** 1. Shift: entry shift when there are free entries behind */ 713 val move = Wire(Vec(nPrefetchEntries, Bool())) 714 (0 until nPrefetchEntries).map (i => 715 // shift when there are invalid (free) entry behind 716 move(i) := ParallelOR((i+1 until nPrefetchEntries).map(!PIQ(_).valid) :+ !handleEntry.valid) 717 ) 718 (1 until nPrefetchEntries).map (i => 719 when(move(i-1)) { 720 PIQ(i) := PIQ(i-1) 721 } 722 ) 723 724 /** 2. Enqueue: receive prefetch request to PIQ(0) when move(0) otherwise blocked*/ 725 enqueReq.ready := move(0) 726 when(move(0)) { 727 when(enqueReq.valid) { 728 PIQ(0).valid := true.B 729 PIQ(0).vSetIdx := enqueReq.bits.vSetIdx 730 PIQ(0).paddr := enqueReq.bits.paddr 731 }.otherwise { 732 PIQ(0).valid := false.B 733 } 734 } 735 736 /** 3. Free: invalid the entry hitted mainpipe */ 737 (0 until PortNumber) .map (i => 738 when(r_hit_PIQ(i)) { 739 when(move(curr_hit_ptr(i)) && (curr_hit_ptr(i) =/= (nPrefetchEntries-1).U)) { 740 // the hitted entry will be shifted, so invalid next entry 741 PIQ(curr_hit_ptr(i) + 1.U).valid := false.B 742 }.otherwise { 743 PIQ(curr_hit_ptr(i)).valid := false.B 744 } 745 } 746 ) 747 748 /** 4. Fencei: invalid all PIQ entry when fencei */ 749 when(io.fencei) { 750 PIQ.map ( 751 _.valid := false.B 752 ) 753 } 754 755 /** 5. Dequeue: move PIQ.last to handle entry when handle entry is free */ 756 // cancel entry when PIQ.last is hitted by mainpipe 757 val cancel = (0 until PortNumber).map (i => r_hit_PIQ(i) && (curr_hit_ptr(i) === (nPrefetchEntries-1).U)).reduce(_||_) 758 when(move(nPrefetchEntries - 1) && !cancel) { 759 handleEntry.valid := PIQ(nPrefetchEntries - 1).valid 760 handleEntry.paddr := PIQ(nPrefetchEntries - 1).paddr 761 handleEntry.vSetIdx := PIQ(nPrefetchEntries - 1).vSetIdx 762 handleEntry.issue := false.B 763 handleEntry.finish := false.B 764 handleEntry.flash := false.B 765 handleEntry.readBeatCnt := 0.U 766 } 767 768 /** 6. Cancel handleEntry hitted mainpipe or receive fencei if hasn't been issued */ 769 when(cancelHandleEntry) { 770 handleEntry.valid := false.B 771 } 772 773 /** 7. Valid flash of handleEntry when fencei. Drop data from TileLink port */ 774 when(io.fencei) { 775 handleEntry.flash := true.B 776 } 777 778 779 /** 780 ****************************************************************************** 781 * Write Prefetch Data to Prefetch Buffer 782 * - valid signal only hold one cycle 783 ****************************************************************************** 784 */ 785 toIPF.valid := handleEntry.valid && handleEntry.finish 786 toIPF.bits.paddr := handleEntry.paddr 787 toIPF.bits.vSetIdx := handleEntry.vSetIdx 788 toIPF.bits.cacheline := handleEntry.cacheline 789 toIPF.bits.has_hit := piq_hit_req 790 791 792 /** 793 ****************************************************************************** 794 * Tilelink Transition 795 * - 1. Send acquire to L2Cache, after which the request cannot be canceled (issue valid) 796 * - 2. Wait data response from Tilelink 797 * - 3. Get data, valid finish (only one cycle) and invalid issue 798 ****************************************************************************** 799 */ 800 val (_, _, refill_done, refill_address_inc) = edge.addr_inc(io.mem_grant) 801 802 val s_idle :: s_memReadReq :: s_memReadResp :: s_write_back :: Nil = Enum(4) 803 val state = RegInit(s_idle) 804 805 // acquire port 806 io.mem_acquire.bits := edge.Get( 807 fromSource = PortNumber.U, 808 toAddress = Cat(handleEntry.paddr(PAddrBits - 1, log2Ceil(blockBytes)), 0.U(log2Ceil(blockBytes).W)), 809 lgSize = (log2Up(cacheParams.blockBytes)).U)._2 810 io.mem_acquire.bits.user.lift(PreferCacheKey).foreach(_ := true.B) 811 io.mem_acquire.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.L1InstPrefetch.id.U) 812 813 // grant port 814 io.mem_grant.ready := true.B 815 io.mem_acquire.valid := (state === s_memReadReq) 816 817 switch(state) { 818 is(s_idle) { 819 when(handleEntry.valid && !cancelHandleEntry) { 820 handleEntry.issue := true.B 821 state := s_memReadReq 822 } 823 } 824 is(s_memReadReq) { 825 state := Mux(io.mem_acquire.fire(), s_memReadResp, s_memReadReq) 826 } 827 is(s_memReadResp) { 828 when (edge.hasData(io.mem_grant.bits) && io.mem_grant.fire()) { 829 handleEntry.readBeatCnt := handleEntry.readBeatCnt + 1.U 830 handleEntry.respData(handleEntry.readBeatCnt) := io.mem_grant.bits.data 831 when (handleEntry.readBeatCnt === (refillCycles - 1).U) { 832 assert(refill_done, "refill not done!") 833 state := s_write_back 834 when (!io.fencei && !handleEntry.flash) { 835 handleEntry.issue := false.B 836 handleEntry.finish := true.B 837 } 838 } 839 } 840 } 841 is(s_write_back) { 842 state := s_idle 843 handleEntry.valid := false.B 844 } 845 } 846 847 if (env.EnableDifftest) { 848 val diffipfrefill = DifftestModule(new DiffRefillEvent, dontCare = true) 849 diffipfrefill.coreid := io.hartId 850 diffipfrefill.index := 3.U 851 diffipfrefill.valid := handleEntry.valid && handleEntry.finish 852 diffipfrefill.addr := handleEntry.paddr 853 diffipfrefill.data := handleEntry.cacheline.asTypeOf(diffipfrefill.data) 854 } 855} 856 857class FDIPPrefetchIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 858 /** commen */ 859 val fencei = Input(Bool()) 860 val hartId = Input(UInt(8.W)) 861 862 /** Prefetch Mainpipe IO */ 863 val ftqReq = Flipped(new FtqPrefechBundle) 864 val iTLBInter = new TlbRequestIO 865 val pmp = new ICachePMPBundle 866 val metaReadReq = Decoupled(new ICacheMetaReadReqBundle) 867 val metaReadResp = Input(new ICacheMetaReadRespBundle) 868 val mshrInfo = Flipped(Vec(PortNumber, ValidIO(UInt(PAddrBits.W)))) 869 val mainPipeMissInfo = Flipped(Vec(PortNumber,ValidIO(new MainPipeMissInfo))) 870 871 /** Prefetch Buffer IO */ 872 val IPFBufferRead = new IPFBufferRead 873 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 874 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 875 val IPFReplacer = new IPFReplacer 876 877 /** Prefetch Queue IO */ 878 val PIQRead = new PIQRead 879 val mem_acquire = new DecoupledIO(new TLBundleA(edge.bundle)) 880 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 881} 882 883class FDIPPrefetch(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule 884{ 885 val io = IO(new FDIPPrefetchIO(edge)) 886 887 val prefetchPipe = Module(new IPrefetchPipe) 888 val prefetchQueue = Module(new PrefetchQueue(edge)) 889 val prefetchBuffer = Module(new PrefetchBuffer) 890 891 io.metaReadReq <> prefetchPipe.io.metaReadReq 892 io.metaWrite <> prefetchBuffer.io.metaWrite 893 io.dataWrite <> prefetchBuffer.io.dataWrite 894 io.IPFReplacer <> prefetchBuffer.io.IPFReplacer 895 io.mem_acquire <> prefetchQueue.io.mem_acquire 896 io.mem_grant <> prefetchQueue.io.mem_grant 897 898 // outside 899 prefetchPipe.io.fencei <> io.fencei 900 prefetchPipe.io.ftqReq <> io.ftqReq 901 prefetchPipe.io.iTLBInter <> io.iTLBInter 902 prefetchPipe.io.pmp <> io.pmp 903 prefetchPipe.io.metaReadResp <> io.metaReadResp 904 prefetchPipe.io.mshrInfo <> io.mshrInfo 905 prefetchPipe.io.mainPipeMissInfo <> io.mainPipeMissInfo 906 907 // outside 908 prefetchBuffer.io.hartId <> io.hartId 909 prefetchBuffer.io.fencei <> io.fencei 910 prefetchBuffer.io.IPFBufferRead <> io.IPFBufferRead 911 // inside 912 prefetchBuffer.io.IPFFilterRead <> prefetchPipe.io.IPFFilterRead 913 prefetchBuffer.io.IPFBufferWrite <> prefetchQueue.io.IPFBufferWrite 914 915 // outside 916 prefetchQueue.io.hartId <> io.hartId 917 prefetchQueue.io.fencei <> io.fencei 918 prefetchQueue.io.PIQRead <> io.PIQRead 919 // inside 920 prefetchQueue.io.prefetchReq <> prefetchPipe.io.prefetchReq 921 prefetchQueue.io.PIQFilterRead <> prefetchPipe.io.PIQFilterRead 922} 923