1/*************************************************************************************** 2 * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3 * Copyright (c) 2020-2021 Peng Cheng Laboratory 4 * 5 * XiangShan is licensed under Mulan PSL v2. 6 * You can use this software according to the terms and conditions of the Mulan PSL v2. 7 * You may obtain a copy of Mulan PSL v2 at: 8 * http://license.coscl.org.cn/MulanPSL2 9 * 10 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13 * 14 * See the Mulan PSL v2 for more details. 15 ***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import difftest._ 23import freechips.rocketchip.tilelink._ 24import utils._ 25import xiangshan.cache.mmu._ 26import xiangshan.frontend._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28import huancun.PreferCacheKey 29import xiangshan.XSCoreParamsKey 30import utility._ 31 32 33abstract class IPrefetchBundle(implicit p: Parameters) extends ICacheBundle 34abstract class IPrefetchModule(implicit p: Parameters) extends ICacheModule 35 36//TODO: remove this 37object DebugFlags { 38 val fdip = false 39} 40 41class PIQData(implicit p: Parameters) extends IPrefetchBundle { 42 val ptage = UInt(tagBits.W) 43 val vSetIdx = UInt(idxBits.W) 44 val cacheline = UInt(blockBits.W) 45 val writeBack = Bool() 46} 47 48class PIQToMainPipe(implicit p: Parameters) extends IPrefetchBundle{ 49 val info = DecoupledIO(new PIQData) 50} 51 52class PrefetchReq(implicit p: Parameters) extends IPrefetchBundle{ 53 val paddr = UInt(PAddrBits.W) 54 val vSetIdx = UInt(idxBits.W) 55} 56 57class PrefetchBufferIO(implicit p: Parameters) extends IPrefetchBundle { 58 val hartId = Input(UInt(hartIdLen.W)) 59 val fencei = Input(Bool()) 60 val IPFFilterRead = new IPFBufferFilterRead 61 val IPFBufferRead = new IPFBufferRead 62 val IPFBufferWrite = Flipped(DecoupledIO(new IPFBufferWrite)) 63 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 64 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 65 val IPFReplacer = new IPFReplacer 66 val ipfRecentWrite = Output(Vec(2, new FilterInfo)) 67} 68 69class PrefetchBuffer(implicit p: Parameters) extends IPrefetchModule 70{ 71 val io = IO(new PrefetchBufferIO) 72 73 val fromIPrefetch = io.IPFFilterRead.req 74 val toIPrefetch = io.IPFFilterRead.resp 75 val fromMainPipe = io.IPFBufferRead.req 76 val toMainPipe = io.IPFBufferRead.resp 77 val fromPIQ = io.IPFBufferWrite 78 val toICacheMeta = io.metaWrite 79 val toICacheData = io.dataWrite 80 val fromReplacer = io.IPFReplacer 81 val toIPrefetchInfo = io.ipfRecentWrite 82 83 class IPFBufferEntryMeta(implicit p: Parameters) extends IPrefetchBundle 84 { 85 val valid = Bool() 86 val paddr = UInt(PAddrBits.W) 87 val vSetIdx = UInt(idxBits.W) 88 val confidence = UInt(log2Ceil(maxIPFMoveConf + 1).W) 89 } 90 91 class IPFBufferEntryData(implicit p: Parameters) extends IPrefetchBundle 92 { 93 val cacheline = UInt(blockBits.W) 94 } 95 96 val meta_buffer = InitQueue(new IPFBufferEntryMeta, size = nPrefBufferEntries) 97 val data_buffer = InitQueue(new IPFBufferEntryData, size = nPrefBufferEntries) 98 99 /** 100 ****************************************************************************** 101 * handle look-up request from IPrefetchPipe 102 * - 1. Receive request from IPrefetch 103 * - 2. Look up whether hit in meta_buffer 104 * - 3. Send response to IPrefetch at the same cycle 105 ****************************************************************************** 106 */ 107 val fr_block = get_block(fromIPrefetch.paddr) 108 val fr_hit_oh = meta_buffer.map(e => e.valid && (get_block(e.paddr) === fr_block)) 109 toIPrefetch.ipf_hit := ParallelOR(fr_hit_oh) 110 // only hit one entry in meta_buffer at the same time 111 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 prefetch buffer for filter") 112 113 /** 114 ****************************************************************************** 115 * handle read request from ICacheMainPipe 116 * - 1. Receive request from ICacheMainPipe 117 * - 2. Look up whether hit in meta_buffer and get hitted data 118 * - 3. Send response to ICacheMainPipe at the same cycle 119 ****************************************************************************** 120 */ 121 fromMainPipe.foreach(_.ready := DontCare) 122 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 123 val r_hit_oh = r_blocks.map (ptag => meta_buffer.map(e => e.valid && (get_block(e.paddr) === ptag))) 124 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 125 val r_hit_data = r_hit_oh.map (oh => Mux1H(oh, data_buffer.map(_.cacheline))) 126 val r_hit = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 127 (0 until PortNumber).foreach (i => { 128 toMainPipe(i).ipf_hit := r_hit(i) 129 toMainPipe(i).cacheline := r_hit_data(i) 130 // only hit one entry in meta_buffer 131 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 prefetch buffer") 132 XSPerfAccumulate("fdip_mainpipe_hit_in_ipf_" + i, r_hit(i)) 133 }) 134 135 136 /** 137 ****************************************************************************** 138 * move the entry arriving max confidence to icache 139 * - 1. Chose a moved entry from the entries that reach max confidence (LSB first) 140 * - 2. Get victim way of array from replacer of ICacheMainPipe 141 * - 3. Send write req to metaArray and dataArray and ensure fire together 142 ****************************************************************************** 143 */ 144 val move_oh = meta_buffer.map (e => e.confidence === maxIPFMoveConf.U && e.valid) 145 val move_need = ParallelOR(move_oh) 146 val curr_move_ptr = PriorityEncoder(move_oh) 147 148 fromReplacer.vsetIdx := meta_buffer(curr_move_ptr).vSetIdx 149 150 toICacheMeta.valid := move_need 151 toICacheMeta.bits.generate( 152 tag = get_phy_tag(meta_buffer(curr_move_ptr).paddr), 153 idx = meta_buffer(curr_move_ptr).vSetIdx, 154 waymask = fromReplacer.waymask, 155 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0)) 156 157 toICacheData.valid := move_need 158 toICacheData.bits.generate( 159 data = data_buffer(curr_move_ptr).cacheline, 160 idx = meta_buffer(curr_move_ptr).vSetIdx, 161 waymask = fromReplacer.waymask, 162 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0), 163 paddr = meta_buffer(curr_move_ptr).paddr) 164 165 // TODO: how to ensure 166 assert((toICacheMeta.fire && toICacheData.fire) || (!toICacheMeta.fire && !toICacheData.fire), 167 "meta and data array need fire at same time") 168 169 XSPerfAccumulate("fdip_move_to_icache", toICacheMeta.fire && toICacheData.fire) 170 171 when(toICacheMeta.fire) { 172 XSDebug(p"PrefetchBuffer: write prefetch data to icache . vSetIdx:${meta_buffer(curr_move_ptr).vSetIdx} " + 173 p"paddr:${meta_buffer(curr_move_ptr).paddr}") 174 } 175 176 /** 177 ****************************************************************************** 178 * Prefetch buffer write logic 179 * Read old data when read and write occur in the same cycle 180 * There are 4 situations that can cause meta_buffer and data_buffer to be written 181 * - 1. ICacheMainPipe read hit that need to change some status 182 * - 2. Move prefetch cacheline to icache 183 * - 3. Receive write request from ICacheMissUnit 184 * - 4. Receive fencei req that need to flush all buffer 185 * Priority: 4 > 3 > 2 > 1 186 * The order of the code determines the priority, with the following priority being higher 187 ****************************************************************************** 188 */ 189 190 /** 1. confidence++ for every ICacheMainPipe hit */ 191 (0 until PortNumber) .map (i => 192 when(r_hit(i)) { 193 // handle overflow 194 when(meta_buffer(curr_hit_ptr(i)).confidence =/= maxIPFMoveConf.U) { 195 meta_buffer(curr_hit_ptr(i)).confidence := meta_buffer(curr_hit_ptr(i)).confidence + 1.U 196 } 197 } 198 ) 199 200 /** 2. set entry invalid after move to icache */ 201 when(toICacheMeta.fire) { 202 meta_buffer(curr_move_ptr).valid := false.B 203 } 204 205 /** 3. write prefetch data to entry */ 206 // try to get a free entry ptr 207 val free_oh = meta_buffer.map (_.valid === false.B) 208 val has_free = ParallelOR(free_oh) 209 val ptr_from_free = PriorityEncoder(free_oh) 210 // get a ptr from random replacement policy 211 val replacer = ReplacementPolicy.fromString(Some("random"), nPrefBufferEntries) 212 // write to free entry if has, otherwise random entry 213 val curr_write_ptr = Mux(has_free, ptr_from_free, replacer.way) 214 fromPIQ.ready := DontCare 215 when(fromPIQ.valid) { 216 meta_buffer(curr_write_ptr).valid := true.B 217 meta_buffer(curr_write_ptr).confidence := fromPIQ.bits.has_hit.asUInt 218 meta_buffer(curr_write_ptr).paddr := fromPIQ.bits.paddr 219 meta_buffer(curr_write_ptr).vSetIdx := fromPIQ.bits.vSetIdx 220 data_buffer(curr_write_ptr).cacheline := fromPIQ.bits.cacheline 221 } 222 223 if(DebugFlags.fdip) { 224 when(fromPIQ.valid) { 225 printf(p"PrefetchBuffer: receive prefetch data. vSetIdx:${fromPIQ.bits.vSetIdx} paddr:${fromPIQ.bits.paddr}") 226 } 227 } 228 229 XSPerfAccumulate("fdip_vitcm_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence =/= 0.U)) 230 XSPerfAccumulate("fdip_vitcm_no_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence === 0.U)) 231 232 /** 4. flush all prefetch buffer when fencei */ 233 when(io.fencei) { 234 meta_buffer.foreach { b => 235 b.valid := false.B 236 b.confidence := 0.U 237 } 238 } 239 240 /** 241 ****************************************************************************** 242 * Register 2 cycle meta write info for IPrefetchPipe filter 243 ****************************************************************************** 244 */ 245 val meta_write_buffer = InitQueue(new FilterInfo, size = 2) 246 meta_write_buffer(0).valid := toICacheMeta.fire 247 meta_write_buffer(0).paddr := meta_buffer(curr_move_ptr).paddr 248 meta_write_buffer(1) := meta_write_buffer(0) 249 (0 until PortNumber).foreach (i => { 250 toIPrefetchInfo(i) := meta_write_buffer(i) 251 }) 252 253 XSPerfAccumulate("fdip_fencei_cycle", io.fencei) 254 255 if (env.EnableDifftest) { 256 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 257 difftest.coreid := io.hartId 258 difftest.index := 6.U 259 difftest.valid := toICacheData.fire 260 difftest.addr := toICacheData.bits.paddr 261 difftest.data := toICacheData.bits.data.asTypeOf(difftest.data) 262 difftest.idtfr := DontCare 263 } 264} 265 266class IPredfetchIO(implicit p: Parameters) extends IPrefetchBundle { 267 val ftqReq = Flipped(new FtqPrefechBundle) 268 val iTLBInter = new TlbRequestIO 269 val pmp = new ICachePMPBundle 270 val metaReadReq = Decoupled(new PrefetchMetaReadBundle) 271 val metaReadResp = Input(new PrefetchMetaRespBundle) 272 val prefetchReq = DecoupledIO(new PrefetchReq) 273 274 val IPFFilterRead = Flipped(new IPFBufferFilterRead) 275 val PIQFilterRead = Flipped(new PIQFilterRead) 276 277 val ipfRecentWrite = Input(Vec(2, new FilterInfo)) 278 val ICacheMissUnitInfo = Flipped(new ICacheMissUnitInfo) 279 val ICacheMainPipeInfo = Flipped(new ICacheMainPipeInfo) 280 281 val fencei = Input(Bool()) 282} 283 284class IPrefetchPipe(implicit p: Parameters) extends IPrefetchModule 285{ 286 val io = IO(new IPredfetchIO) 287 288 val enableBit = RegInit(false.B) 289 enableBit := enableICachePrefetch.B 290 291 val fromFtq = io.ftqReq 292 val (toITLB, fromITLB) = (io.iTLBInter.req, io.iTLBInter.resp) 293 val (toIMeta, fromIMeta, fromIMetaValid) = (io.metaReadReq, io.metaReadResp.metaData, io.metaReadResp.entryValid) 294 val (toIPFBuffer, fromIPFBuffer) = (io.IPFFilterRead.req, io.IPFFilterRead.resp) 295 val (toPIQ, fromPIQ) = (io.PIQFilterRead.req, io.PIQFilterRead.resp) 296 val (toPMP, fromPMP) = (io.pmp.req, io.pmp.resp) 297 val toPIQEnqReq = io.prefetchReq 298 299 val fromIPFInfo = io.ipfRecentWrite 300 val fromMainPipeInfo = io.ICacheMainPipeInfo 301 val fromMissUnitInfo = io.ICacheMissUnitInfo 302 303 val p0_fire, p1_fire, p2_fire = WireInit(false.B) 304 val p0_discard, p1_discard, p2_discard = WireInit(false.B) 305 val p0_ready, p1_ready, p2_ready = WireInit(false.B) 306 307 /** 308 ****************************************************************************** 309 * IPrefetch Stage 0 310 * - 1. send req to IMeta 311 * - 2. send req to ITLB (no blocked) 312 * - 3. check last req 313 ****************************************************************************** 314 */ 315 val p0_valid = fromFtq.req.valid 316 317 val p0_vaddr = addrAlign(fromFtq.req.bits.target, blockBytes, VAddrBits) 318 val p0_vaddr_reg = RegEnable(p0_vaddr, p0_fire) 319 val p0_req_cancel = Wire(Bool()) 320 321 /** 1. send req to IMeta */ 322 toIMeta.valid := p0_valid && !p0_req_cancel 323 toIMeta.bits.idx := get_idx(p0_vaddr) 324 325 /** 2. send req to ITLB (no blocked) */ 326 toITLB.valid := p0_valid && !p0_req_cancel 327 toITLB.bits.size := 3.U // TODO: fix the size 328 toITLB.bits.vaddr := p0_vaddr 329 toITLB.bits.debug.pc := p0_vaddr 330 toITLB.bits.kill := DontCare 331 toITLB.bits.cmd := TlbCmd.exec 332 toITLB.bits.debug.robIdx := DontCare 333 toITLB.bits.debug.isFirstIssue := DontCare 334 toITLB.bits.memidx := DontCare 335 toITLB.bits.no_translate := false.B 336 fromITLB.ready := true.B 337 // TODO: whether to handle tlb miss for prefetch 338 io.iTLBInter.req_kill := true.B 339 340 /** FTQ request port */ 341 fromFtq.req.ready := p0_ready 342 343 /** 3. Check last req: request from FTQ is same as last time or behind */ 344 val p0_hit_behind = Wire(Bool()) 345 346 /** stage 0 control */ 347 // Cancel request when prefetch not enable or the request from FTQ is same as last time 348 p0_req_cancel := !enableBit || p0_hit_behind || io.fencei 349 p0_ready := p0_req_cancel || p1_ready && toITLB.ready && toIMeta.ready 350 p0_fire := p0_valid && p1_ready && toITLB.ready && toIMeta.ready && enableBit && !p0_req_cancel 351 p0_discard := p0_valid && p0_req_cancel 352 353 if(DebugFlags.fdip) { 354 when(p0_discard) { 355 printf(p"IPrefetchPipe: discard req in s0.") 356 } 357 } 358 359 /** 360 ****************************************************************************** 361 * IPrefetch Stage 1 362 * - 1. Receive resp from ITLB (no blocked) 363 * - 2. Receive resp from IMeta 364 * - 3. Check MSHR 365 ****************************************************************************** 366 */ 367 val p1_valid = generatePipeControl(lastFire = p0_fire, thisFire = p1_fire || p1_discard, thisFlush = false.B, lastFlush = false.B) 368 369 val p1_vaddr = RegEnable(p0_vaddr, 0.U(VAddrBits.W), p0_fire) 370 val p1_req_cancel = Wire(Bool()) 371 372 /** 1. Receive resp from ITLB (no blocked) */ 373 val p1_tlb_resp_miss = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.miss) 374 val p1_tlb_resp_paddr = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.paddr(0)) 375 val p1_tlb_resp_pf = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).pf.instr) 376 val p1_tlb_resp_af = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).af.instr) 377 val p1_exception = VecInit(Seq(p1_tlb_resp_pf, p1_tlb_resp_af)) 378 val p1_has_except = p1_exception.reduce(_ || _) 379 val p1_paddr = p1_tlb_resp_paddr 380 381 /** 2. Receive resp from IMeta. Register the data first because of strict timing. */ 382 val p1_meta_ptags_reg = RegEnable(VecInit(fromIMeta.map(way => way.tag)), RegNext(p0_fire)) 383 val p1_meta_valids_reg = RegEnable(fromIMetaValid, RegNext(p0_fire)) 384 385 /** Stage 1 control */ 386 p1_req_cancel := p1_tlb_resp_miss || p1_has_except || io.fencei 387 p1_ready := p1_valid && p2_ready || !p1_valid 388 p1_fire := p1_valid && !p1_req_cancel && p2_ready && enableBit 389 p1_discard := p1_valid && p1_req_cancel 390 391 when(p1_discard) { 392 XSDebug(p"IPrefetchPipe: discard req in s1. vaddr:${p1_vaddr}") 393 } 394 395 /** 396 ****************************************************************************** 397 * IPrefetch Stage 2 398 * - 1. IMeta Check: check whether req hit in icache 399 * - 2. PMP Check: send req and receive resq in the same cycle 400 * - 3. Prefetch buffer Check (if prefetch to L1) 401 * - 4. Prefetch Queue Check 402 * - 5. Prefetch Buffer Recently write chcek 403 * - 6. MainPipeInfo chcek 404 * - 7. MissUnitInfo chcek 405 * - 8. Recently meta write check 406 ****************************************************************************** 407 */ 408 val p2_valid = generatePipeControl(lastFire = p1_fire, thisFire = p2_fire || p2_discard, thisFlush = false.B, lastFlush = false.B) 409 410 val p2_paddr = RegEnable(p1_paddr, p1_fire) 411 val p2_vaddr = RegEnable(p1_vaddr, 0.U(VAddrBits.W), p1_fire) 412 val p2_req_cancel = Wire(Bool()) 413 val p2_vidx = get_idx(p2_vaddr) 414 415 p0_hit_behind := (p0_vaddr === p1_vaddr) || (p0_vaddr === p2_vaddr) 416 417 /** 1. IMeta Check */ 418 val p2_ptag = get_phy_tag(p2_paddr) 419 val p2_tag_eq_vec = VecInit(p1_meta_ptags_reg.map(_ === p2_ptag )) 420 val p2_tag_match_vec = VecInit(p2_tag_eq_vec.zipWithIndex.map{ case(way_tag_eq, w) => way_tag_eq && p1_meta_valids_reg(w)}) 421 val p2_tag_match = DataHoldBypass(ParallelOR(p2_tag_match_vec), RegNext(p1_fire)) 422 423 /** 2. PMP Check */ 424 // TODO: When fromPMP.mmio is low but fromPMP.instr is high, which is a except? 425 val p2_exception = DataHoldBypass((fromPMP.mmio && !fromPMP.instr) || fromPMP.instr, RegNext(p1_fire)) 426 // PMP request 427 toPMP.valid := p2_valid 428 toPMP.bits.addr := p2_paddr 429 toPMP.bits.size := 3.U 430 toPMP.bits.cmd := TlbCmd.exec 431 432 /** 3. Prefetch buffer Check */ 433 toIPFBuffer.paddr := p2_paddr 434 val p2_hit_buffer = fromIPFBuffer.ipf_hit 435 436 /** 4. Prefetch Queue Check */ 437 toPIQ.paddr := p2_paddr 438 val p2_hit_piq = fromPIQ.piq_hit 439 440 /** 5. Prefetch Buffer Recently write chcek */ 441 val p2_check_ipf_info = VecInit(fromIPFInfo.map(info => 442 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 443 444 /** 6. MainPipeInfo chcek */ 445 val check_mp_s1 = VecInit(fromMainPipeInfo.s1Info.map(info => 446 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 447 val check_mp_s2 = VecInit(fromMainPipeInfo.s2Info.map(info => 448 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 449 val check_mp_missSlot = VecInit(fromMainPipeInfo.missSlot.map(info => 450 info.valid && info.ptag === get_phy_tag(p2_paddr) && info.vSetIdx === p2_vidx)).reduce(_||_) 451 val p2_check_mp_info = check_mp_s1 || check_mp_s2 || check_mp_missSlot 452 453 /** 7. MissUnitInfo chcek */ 454 val check_mu_mshr = VecInit(fromMissUnitInfo.mshr.map(info => 455 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 456 val check_mu_recent_write = VecInit(fromMissUnitInfo.recentWrite.map(info => 457 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 458 val p2_check_mu_info = check_mu_mshr || check_mu_recent_write 459 460 /** 8. send req to piq */ 461 toPIQEnqReq.valid := p2_valid && !p2_req_cancel 462 toPIQEnqReq.bits.paddr := p2_paddr 463 toPIQEnqReq.bits.vSetIdx := p2_vidx 464 465 /** Stage 2 control */ 466 p2_req_cancel := p2_tag_match || p2_exception || p2_hit_buffer || p2_hit_piq || 467 p2_check_ipf_info || p2_check_mp_info || p2_check_mu_info || io.fencei 468 p2_ready := p2_valid && toPIQEnqReq.ready || !p2_valid 469 p2_fire := toPIQEnqReq.fire 470 p2_discard := p2_valid && p2_req_cancel 471 472 when(p2_discard) { 473 XSDebug(p"IPrefetchPipe: discard req in s2. vaddr:${p2_vaddr}") 474 } 475 476 when(p2_fire) { 477 XSDebug(p"IPrefetchPipe: send req to PIQ. vaddr:${p2_vaddr} paddr:${p2_paddr}") 478 } 479 480 /** PerfAccumulate */ 481 // the number of prefetch request received from ftq 482 XSPerfAccumulate("prefetch_req_receive", fromFtq.req.fire) 483 // the number of prefetch request sent to PIQ 484 XSPerfAccumulate("prefetch_req_send_piq", toPIQEnqReq.fire) 485 /** 486 * Count the number of requests that are filtered for various reasons. 487 * The number of prefetch discard in Performance Accumulator may be 488 * a littel larger the number of really discarded. Because there can 489 * be multiple reasons for a canceled request at the same time. 490 */ 491 // discard prefetch request by fencei instruction 492 XSPerfAccumulate("fdip_prefetch_discard_by_fencei", (p0_discard || p1_discard || p2_discard) && io.fencei) 493 494 // discard prefetch request by the request is same as pipeline behind 495 XSPerfAccumulate("fdip_prefetch_discard_by_behind", p0_discard && p0_hit_behind) 496 // discard prefetch request by tlb miss 497 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_miss", p1_discard && p1_tlb_resp_miss) 498 // discard prefetch request by tlb except 499 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_except", p1_discard && p1_has_except) 500 // discard prefetch request by hit mainPipe info 501 XSPerfAccumulate("fdip_prefetch_discard_by_mainPipe", p2_discard && p2_check_mp_info) 502 // discard prefetch request by hit missUnit info 503 XSPerfAccumulate("fdip_prefetch_discard_by_missUnit", p2_discard && p2_check_mu_info) 504 // discard prefetch request by hit icache 505 XSPerfAccumulate("fdip_prefetch_discard_by_hit_cache", p2_discard && p2_tag_match) 506 // discard prefetch request by pmp except or mmio 507 XSPerfAccumulate("fdip_prefetch_discard_by_pmp", p2_discard && p2_exception) 508 // discard prefetch request by hit in prefetch buffer 509 XSPerfAccumulate("fdip_prefetch_discard_by_hit_pf", p2_discard && p2_hit_buffer) 510 // discard prefetch request by hit in prefetch queue 511 XSPerfAccumulate("fdip_prefetch_discard_by_hit_piq", p2_discard && p2_hit_piq) 512} 513 514class PiqPtr(implicit p: Parameters) extends CircularQueuePtr[PiqPtr]( 515 p => p(XSCoreParamsKey).icacheParameters.nPrefetchEntries 516){ 517} 518 519object PiqPtr { 520 def apply(f: Bool, v: UInt)(implicit p: Parameters): PiqPtr = { 521 val ptr = Wire(new PiqPtr) 522 ptr.flag := f 523 ptr.value := v 524 ptr 525 } 526 def inverse(ptr: PiqPtr)(implicit p: Parameters): PiqPtr = { 527 apply(!ptr.flag, ptr.value) 528 } 529} 530 531class PrefetchQueueIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 532 val hartId = Input(UInt(8.W)) 533 val prefetchReq = Flipped(DecoupledIO(new PrefetchReq)) 534 val PIQFilterRead = new PIQFilterRead 535 val PIQRead = new PIQRead 536 val IPFBufferWrite = DecoupledIO(new IPFBufferWrite) 537 /** TileLink Port */ 538 val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 539 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 540 val fencei = Input(Bool()) 541} 542 543class PrefetchQueue(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule with HasCircularQueuePtrHelper { 544 val io = IO(new PrefetchQueueIO(edge)) 545 546 val enqueReq = io.prefetchReq 547 val fromIPrefetch = io.PIQFilterRead.req 548 val toIPrefetch = io.PIQFilterRead.resp 549 val fromMainPipe = io.PIQRead.req 550 val toMainPipe = io.PIQRead.resp 551 val toIPF = io.IPFBufferWrite 552 553 /** Define entryt content and initial FIFO queue */ 554 class PrefetchEntry(implicit p: Parameters) extends IPrefetchBundle { 555 val valid = Bool() 556 val paddr = UInt(PAddrBits.W) 557 val vSetIdx = UInt(idxBits.W) 558 } 559 val PIQ = RegInit(VecInit(Seq.fill(nPrefetchEntries)(0.U.asTypeOf((new PrefetchEntry).cloneType)))) 560 561 val enqPtr = RegInit(PiqPtr(false.B, 0.U)) 562 val deqPtr = RegInit(PiqPtr(false.B, 0.U)) 563 564 class HandleEntry(implicit p: Parameters) extends IPrefetchBundle { 565 val valid = Bool() 566 val paddr = UInt(PAddrBits.W) 567 val vSetIdx = UInt(idxBits.W) 568 val issue = Bool() 569 val finish = Bool() 570 val flash = Bool() 571 val readBeatCnt = UInt(log2Up(refillCycles).W) 572 val respData = Vec(refillCycles, UInt(beatBits.W)) 573 574 def cacheline = respData.asUInt 575 } 576 val handleEntry = RegInit(0.U.asTypeOf((new HandleEntry).cloneType)) 577 578 /** 579 ****************************************************************************** 580 * handle look-up request from IPrefetchPipe 581 * - 1. Receive request from IPrefetch 582 * - 2. Look up whether hit in PIQ and handleEntry 583 * - 3. Send response to IPrefetch at the same cycle 584 ****************************************************************************** 585 */ 586 val fr_block = get_block(fromIPrefetch.paddr) 587 val fr_hit_oh = PIQ.map(e => e.valid && (get_block(e.paddr) === fr_block)) :+ 588 (handleEntry.valid && (get_block(handleEntry.paddr) === fr_block)) 589 val fr_hit = ParallelOR(fr_hit_oh) 590 toIPrefetch.piq_hit := fr_hit 591 // only hit one entry in PIQ 592 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 PIQ for filter") 593 594 595 /** 596 ****************************************************************************** 597 * handle read request from ICacheMainPipe 598 * - 1. Receive request from ICacheMainPipe 599 * - 2. Look up PIQ and handleEntry 600 * - 3. Send response to ICacheMainPipe at the same cycle 601 ****************************************************************************** 602 */ 603 fromMainPipe.foreach(_.ready := DontCare) 604 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 605 /** When hit in PIQ, invalid hitted entry */ 606 val r_hit_oh = r_blocks.map (block => PIQ.map(e => e.valid && (get_block(e.paddr) === block))) 607 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 608 val r_hit_PIQ = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 609 // only hit one entry in PIQ 610 (0 until PortNumber).map (i => 611 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 PIQ") 612 ) 613 614 /** When hit in handleEntry has't been issued, cancel it and return not hit at the same cycle. 615 * When hit in handleEntry has been issued return hit at the same cycle, then return data when transiton finish. 616 * ICacheMainPipe should be blocked during handle handleEntry. 617 */ 618 val r_hit_handle = r_blocks.map(block => handleEntry.valid && (block === get_block(handleEntry.paddr))) 619 val cancelHandleEntry = (((0 until PortNumber).map(i=> fromMainPipe(i).valid && r_hit_handle(i))).reduce(_||_) || io.fencei) && !handleEntry.issue 620 val piq_hit = (0 until PortNumber).map(i => r_hit_handle(i) && handleEntry.issue && !handleEntry.flash && !io.fencei) 621 toMainPipe := DontCare 622 if (prefetchToL1) { 623 (0 until PortNumber).foreach (i => { 624 toMainPipe(i).piq_hit := piq_hit(i) 625 toMainPipe(i).cacheline := handleEntry.cacheline 626 toMainPipe(i).data_valid := handleEntry.valid && handleEntry.finish 627 }) 628 } 629 val piq_hit_req = RegEnable(piq_hit.reduce(_||_), fromMainPipe.map(_.valid).reduce(_||_)) 630 631 XSPerfAccumulate("cancel_req_by_mainpipe_hit_handle", cancelHandleEntry && !io.mem_acquire.fire) 632 633 /** 634 ****************************************************************************** 635 * Manage prefetch requests by queue 636 * - 1. Free: invalid the entry hitted mainpipe 637 * - 2. Dequeue: move req to handle entry when handle entry is free 638 * - 3. Enqueue: enqueue new req, flush earlier req when PIQ is full 639 * - 4. Fencei: invalid all PIQ entry when fencei 640 * - 5. Dequeue: move prefetch request from last entry to handle entry 641 * - 6. Cancel handleEntry hitted mainpipe if hasn't been issued 642 * - 7. Valid flash of handleEntry when fencei if has been issued 643 ****************************************************************************** 644 */ 645 /** 1. Free: invalid the entry hitted mainpipe */ 646 (0 until PortNumber) .foreach (i => { 647 when(r_hit_PIQ(i)) { 648 PIQ(curr_hit_ptr(i)).valid := false.B 649 } 650 XSPerfAccumulate("cancel_req_by_mainpipe_hit_" + i.toString(), r_hit_PIQ(i)) 651 }) 652 653 /** 2. Dequeue: move req to handle entry when handle entry is free */ 654 // cancel entry when dequene req is hitted by mainpipe 655 val cancel = (0 until PortNumber).map (i => r_hit_PIQ(i) && (curr_hit_ptr(i) === deqPtr.value)).reduce(_||_) 656 when(!handleEntry.valid && !cancel && !isEmpty(enqPtr, deqPtr)) { 657 handleEntry.valid := PIQ(deqPtr.value).valid 658 handleEntry.paddr := PIQ(deqPtr.value).paddr 659 handleEntry.vSetIdx := PIQ(deqPtr.value).vSetIdx 660 handleEntry.issue := false.B 661 handleEntry.finish := false.B 662 handleEntry.flash := false.B 663 handleEntry.readBeatCnt := 0.U 664 PIQ(deqPtr.value).valid := false.B 665 deqPtr := deqPtr + 1.U 666 } 667 668 /** 3. Enqueue: enqueue new req, flush earlier req when PIQ is full */ 669 enqueReq.ready := true.B 670 when(enqueReq.valid) { 671 PIQ(enqPtr.value).valid := true.B 672 PIQ(enqPtr.value).vSetIdx := enqueReq.bits.vSetIdx 673 PIQ(enqPtr.value).paddr := enqueReq.bits.paddr 674 enqPtr := enqPtr + 1.U 675 when(isFull(enqPtr, deqPtr)) { 676 deqPtr := deqPtr + 1.U 677 } 678 } 679 XSPerfAccumulate("cancel_req_by_piq_full", enqueReq.valid && PIQ(enqPtr.value).valid) 680 681 /** 4. Fencei: invalid all PIQ entry when fencei */ 682 when(io.fencei) { 683 PIQ.map ( 684 _.valid := false.B 685 ) 686 enqPtr.value := 0.U 687 enqPtr.flag := false.B 688 deqPtr.value := 0.U 689 deqPtr.flag := false.B 690 } 691 692 /** 5. Cancel handleEntry hitted mainpipe or receive fencei if hasn't been issued */ 693 when(cancelHandleEntry) { 694 handleEntry.valid := false.B 695 } 696 697 /** 6. Valid flash of handleEntry when fencei. Drop data from TileLink port */ 698 when(io.fencei) { 699 handleEntry.flash := true.B 700 } 701 702 // deqPtr <= enqPtr 703 // assert(deqPtr <= enqPtr, "deqPtr must be not in front of enqPtr!") 704 assert((deqPtr.flag ^ enqPtr.flag) && (deqPtr.value >= enqPtr.value) || (!(deqPtr.flag ^ enqPtr.flag)) && (deqPtr.value <= enqPtr.value), "deqPtr must be not in front of enqPtr!") 705 706 /** 707 ****************************************************************************** 708 * Write Prefetch Data to Prefetch Buffer 709 * - valid signal only hold one cycle 710 ****************************************************************************** 711 */ 712 toIPF.valid := handleEntry.valid && handleEntry.finish 713 toIPF.bits.paddr := handleEntry.paddr 714 toIPF.bits.vSetIdx := handleEntry.vSetIdx 715 toIPF.bits.cacheline := handleEntry.cacheline 716 toIPF.bits.has_hit := piq_hit_req 717 718 719 /** 720 ****************************************************************************** 721 * Tilelink Transition 722 * - 1. Send acquire to L2Cache, after which the request cannot be canceled (issue valid) 723 * - 2. Wait data response from Tilelink 724 * - 3. Get data, valid finish (only one cycle) and invalid issue 725 ****************************************************************************** 726 */ 727 val (_, _, refill_done, refill_address_inc) = edge.addr_inc(io.mem_grant) 728 729 val s_idle :: s_memReadReq :: s_memReadResp :: s_write_back :: Nil = Enum(4) 730 val state = RegInit(s_idle) 731 732 // acquire port 733 io.mem_acquire.bits := DontCare 734 if (prefetchToL1) { 735 io.mem_acquire.bits := edge.Get( 736 fromSource = PortNumber.U, 737 toAddress = Cat(handleEntry.paddr(PAddrBits - 1, log2Ceil(blockBytes)), 0.U(log2Ceil(blockBytes).W)), 738 lgSize = (log2Up(cacheParams.blockBytes)).U)._2 739 } else { 740 io.mem_acquire.bits := edge.Hint( 741 fromSource = PortNumber.U, 742 toAddress = addrAlign(handleEntry.paddr, blockBytes, PAddrBits), 743 lgSize = (log2Up(cacheParams.blockBytes)).U, 744 param = TLHints.PREFETCH_READ)._2 745 } 746 747 io.mem_acquire.bits.user.lift(PreferCacheKey).foreach(_ := true.B) 748 io.mem_acquire.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.L1InstPrefetch.id.U) 749 750 // grant port 751 io.mem_grant.ready := true.B 752 io.mem_acquire.valid := (state === s_memReadReq) 753 754 if (prefetchToL1) { 755 switch(state) { 756 is(s_idle) { 757 when(handleEntry.valid && !cancelHandleEntry) { 758 handleEntry.issue := true.B 759 state := s_memReadReq 760 } 761 } 762 } 763 is(s_memReadReq) { 764 state := Mux(io.mem_acquire.fire, s_memReadResp, s_memReadReq) 765 } 766 is(s_memReadResp) { 767 when (edge.hasData(io.mem_grant.bits) && io.mem_grant.fire) { 768 handleEntry.readBeatCnt := handleEntry.readBeatCnt + 1.U 769 handleEntry.respData(handleEntry.readBeatCnt) := io.mem_grant.bits.data 770 when (handleEntry.readBeatCnt === (refillCycles - 1).U) { 771 assert(refill_done, "refill not done!") 772 state := s_write_back 773 when (!io.fencei && !handleEntry.flash) { 774 handleEntry.issue := false.B 775 handleEntry.finish := true.B 776 } 777 } 778 } 779 is(s_write_back) { 780 state := s_idle 781 handleEntry.valid := false.B 782 } 783 } 784 } else { 785 switch(state) { 786 is(s_idle) { 787 when(handleEntry.valid && !cancelHandleEntry) { 788 state := s_memReadReq 789 } 790 } 791 is(s_memReadReq) { 792 when(io.mem_acquire.fire || cancelHandleEntry) { 793 state := s_idle 794 handleEntry.valid := false.B 795 } otherwise { 796 state := s_memReadReq 797 } 798 } 799 } 800 } 801 802 // the number of prefetch request sent to L2Cache 803 XSPerfAccumulate("prefetch_req_send_L2", io.mem_acquire.fire) 804 805 if (env.EnableDifftest) { 806 val diffipfrefill = DifftestModule(new DiffRefillEvent, dontCare = true) 807 diffipfrefill.coreid := io.hartId 808 diffipfrefill.index := 3.U 809 diffipfrefill.valid := handleEntry.valid && handleEntry.finish 810 diffipfrefill.addr := handleEntry.paddr 811 diffipfrefill.data := handleEntry.cacheline.asTypeOf(diffipfrefill.data) 812 diffipfrefill.idtfr := DontCare 813 } 814} 815 816class FDIPPrefetchIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 817 /** commen */ 818 val fencei = Input(Bool()) 819 val hartId = Input(UInt(hartIdLen.W)) 820 821 /** Prefetch Mainpipe IO */ 822 val ftqReq = Flipped(new FtqPrefechBundle) 823 val iTLBInter = new TlbRequestIO 824 val pmp = new ICachePMPBundle 825 val metaReadReq = Decoupled(new PrefetchMetaReadBundle) 826 val metaReadResp = Input(new PrefetchMetaRespBundle) 827 828 val ICacheMissUnitInfo = Flipped(new ICacheMissUnitInfo) 829 val ICacheMainPipeInfo = Flipped(new ICacheMainPipeInfo) 830 831 /** Prefetch Buffer IO */ 832 val IPFBufferRead = new IPFBufferRead 833 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 834 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 835 val IPFReplacer = new IPFReplacer 836 837 /** Prefetch Queue IO */ 838 val PIQRead = new PIQRead 839 val mem_acquire = new DecoupledIO(new TLBundleA(edge.bundle)) 840 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 841} 842 843class FDIPPrefetch(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule 844{ 845 val io = IO(new FDIPPrefetchIO(edge)) 846 847 val prefetchPipe = Module(new IPrefetchPipe) 848 val prefetchQueue = Module(new PrefetchQueue(edge)) 849 850 io.metaReadReq <> prefetchPipe.io.metaReadReq 851 io.metaWrite <> DontCare 852 io.dataWrite <> DontCare 853 io.IPFReplacer <> DontCare 854 io.IPFBufferRead <> DontCare 855 io.mem_acquire <> prefetchQueue.io.mem_acquire 856 io.mem_grant <> prefetchQueue.io.mem_grant 857 858 // outside 859 prefetchPipe.io.fencei <> io.fencei 860 prefetchPipe.io.ftqReq <> io.ftqReq 861 prefetchPipe.io.iTLBInter <> io.iTLBInter 862 prefetchPipe.io.pmp <> io.pmp 863 prefetchPipe.io.metaReadResp <> io.metaReadResp 864 prefetchPipe.io.ICacheMissUnitInfo <> io.ICacheMissUnitInfo 865 prefetchPipe.io.ICacheMainPipeInfo <> io.ICacheMainPipeInfo 866 // inside 867 prefetchPipe.io.ipfRecentWrite <> DontCare 868 prefetchPipe.io.IPFFilterRead <> DontCare 869 870 // outside 871 prefetchQueue.io.hartId <> io.hartId 872 prefetchQueue.io.fencei <> io.fencei 873 prefetchQueue.io.PIQRead <> io.PIQRead 874 // inside 875 prefetchQueue.io.prefetchReq <> prefetchPipe.io.prefetchReq 876 prefetchQueue.io.PIQFilterRead <> prefetchPipe.io.PIQFilterRead 877 prefetchQueue.io.IPFBufferWrite <> DontCare 878 879 if (prefetchToL1) { 880 val prefetchBuffer = Module(new PrefetchBuffer) 881 // outside 882 prefetchBuffer.io.hartId <> io.hartId 883 prefetchBuffer.io.fencei <> io.fencei 884 prefetchBuffer.io.IPFBufferRead <> io.IPFBufferRead 885 prefetchBuffer.io.metaWrite <> io.metaWrite 886 prefetchBuffer.io.dataWrite <> io.dataWrite 887 prefetchBuffer.io.IPFReplacer <> io.IPFReplacer 888 // inside 889 prefetchBuffer.io.IPFFilterRead <> prefetchPipe.io.IPFFilterRead 890 prefetchBuffer.io.ipfRecentWrite <> prefetchPipe.io.ipfRecentWrite 891 prefetchBuffer.io.IPFBufferWrite <> prefetchQueue.io.IPFBufferWrite 892 } 893}