1/*************************************************************************************** 2 * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3 * Copyright (c) 2020-2021 Peng Cheng Laboratory 4 * 5 * XiangShan is licensed under Mulan PSL v2. 6 * You can use this software according to the terms and conditions of the Mulan PSL v2. 7 * You may obtain a copy of Mulan PSL v2 at: 8 * http://license.coscl.org.cn/MulanPSL2 9 * 10 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13 * 14 * See the Mulan PSL v2 for more details. 15 ***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import difftest._ 23import freechips.rocketchip.tilelink._ 24import utils._ 25import xiangshan.cache.mmu._ 26import xiangshan.frontend._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28import huancun.PreferCacheKey 29import xiangshan.XSCoreParamsKey 30import utility._ 31 32 33abstract class IPrefetchBundle(implicit p: Parameters) extends ICacheBundle 34abstract class IPrefetchModule(implicit p: Parameters) extends ICacheModule 35 36//TODO: remove this 37object DebugFlags { 38 val fdip = false 39} 40 41class PIQData(implicit p: Parameters) extends IPrefetchBundle { 42 val ptage = UInt(tagBits.W) 43 val vSetIdx = UInt(idxBits.W) 44 val cacheline = UInt(blockBits.W) 45 val writeBack = Bool() 46} 47 48class PIQToMainPipe(implicit p: Parameters) extends IPrefetchBundle{ 49 val info = DecoupledIO(new PIQData) 50} 51 52class PrefetchReq(implicit p: Parameters) extends IPrefetchBundle{ 53 val paddr = UInt(PAddrBits.W) 54 val vSetIdx = UInt(idxBits.W) 55} 56 57class PrefetchBufferIO(implicit p: Parameters) extends IPrefetchBundle { 58 val hartId = Input(UInt(hartIdLen.W)) 59 val fencei = Input(Bool()) 60 val IPFFilterRead = new IPFBufferFilterRead 61 val IPFBufferRead = new IPFBufferRead 62 val IPFBufferWrite = Flipped(DecoupledIO(new IPFBufferWrite)) 63 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 64 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 65 val IPFReplacer = new IPFReplacer 66 val ipfRecentWrite = Output(Vec(2, new FilterInfo)) 67} 68 69class PrefetchBuffer(implicit p: Parameters) extends IPrefetchModule 70{ 71 val io = IO(new PrefetchBufferIO) 72 73 val fromIPrefetch = io.IPFFilterRead.req 74 val toIPrefetch = io.IPFFilterRead.resp 75 val fromMainPipe = io.IPFBufferRead.req 76 val toMainPipe = io.IPFBufferRead.resp 77 val fromPIQ = io.IPFBufferWrite 78 val toICacheMeta = io.metaWrite 79 val toICacheData = io.dataWrite 80 val fromReplacer = io.IPFReplacer 81 val toIPrefetchInfo = io.ipfRecentWrite 82 83 class IPFBufferEntryMeta(implicit p: Parameters) extends IPrefetchBundle 84 { 85 val valid = Bool() 86 val paddr = UInt(PAddrBits.W) 87 val vSetIdx = UInt(idxBits.W) 88 val confidence = UInt(log2Ceil(maxIPFMoveConf + 1).W) 89 } 90 91 class IPFBufferEntryData(implicit p: Parameters) extends IPrefetchBundle 92 { 93 val cacheline = UInt(blockBits.W) 94 } 95 96 val meta_buffer = InitQueue(new IPFBufferEntryMeta, size = nPrefBufferEntries) 97 val data_buffer = InitQueue(new IPFBufferEntryData, size = nPrefBufferEntries) 98 99 /** 100 ****************************************************************************** 101 * handle look-up request from IPrefetchPipe 102 * - 1. Receive request from IPrefetch 103 * - 2. Look up whether hit in meta_buffer 104 * - 3. Send response to IPrefetch at the same cycle 105 ****************************************************************************** 106 */ 107 val fr_block = get_block(fromIPrefetch.paddr) 108 val fr_hit_oh = meta_buffer.map(e => e.valid && (get_block(e.paddr) === fr_block)) 109 toIPrefetch.ipf_hit := ParallelOR(fr_hit_oh) 110 // only hit one entry in meta_buffer at the same time 111 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 prefetch buffer for filter") 112 113 /** 114 ****************************************************************************** 115 * handle read request from ICacheMainPipe 116 * - 1. Receive request from ICacheMainPipe 117 * - 2. Look up whether hit in meta_buffer and get hitted data 118 * - 3. Send response to ICacheMainPipe at the same cycle 119 ****************************************************************************** 120 */ 121 fromMainPipe.foreach(_.ready := DontCare) 122 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 123 val r_hit_oh = r_blocks.map (ptag => meta_buffer.map(e => e.valid && (get_block(e.paddr) === ptag))) 124 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 125 val r_hit_data = r_hit_oh.map (oh => Mux1H(oh, data_buffer.map(_.cacheline))) 126 val r_hit = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 127 (0 until PortNumber).foreach (i => { 128 toMainPipe(i).ipf_hit := r_hit(i) 129 toMainPipe(i).cacheline := r_hit_data(i) 130 // only hit one entry in meta_buffer 131 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 prefetch buffer") 132 XSPerfAccumulate("fdip_mainpipe_hit_in_ipf_" + i, r_hit(i)) 133 }) 134 135 136 /** 137 ****************************************************************************** 138 * move the entry arriving max confidence to icache 139 * - 1. Chose a moved entry from the entries that reach max confidence (LSB first) 140 * - 2. Get victim way of array from replacer of ICacheMainPipe 141 * - 3. Send write req to metaArray and dataArray and ensure fire together 142 ****************************************************************************** 143 */ 144 val move_oh = meta_buffer.map (e => e.confidence === maxIPFMoveConf.U && e.valid) 145 val move_need = ParallelOR(move_oh) 146 val curr_move_ptr = PriorityEncoder(move_oh) 147 148 fromReplacer.vsetIdx := meta_buffer(curr_move_ptr).vSetIdx 149 150 toICacheMeta.valid := move_need 151 toICacheMeta.bits.generate( 152 tag = get_phy_tag(meta_buffer(curr_move_ptr).paddr), 153 idx = meta_buffer(curr_move_ptr).vSetIdx, 154 waymask = fromReplacer.waymask, 155 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0)) 156 157 toICacheData.valid := move_need 158 toICacheData.bits.generate( 159 data = data_buffer(curr_move_ptr).cacheline, 160 idx = meta_buffer(curr_move_ptr).vSetIdx, 161 waymask = fromReplacer.waymask, 162 bankIdx = meta_buffer(curr_move_ptr).vSetIdx(0), 163 paddr = meta_buffer(curr_move_ptr).paddr) 164 165 // TODO: how to ensure 166 assert((toICacheMeta.fire && toICacheData.fire) || (!toICacheMeta.fire && !toICacheData.fire), 167 "meta and data array need fire at same time") 168 169 XSPerfAccumulate("fdip_move_to_icache", toICacheMeta.fire && toICacheData.fire) 170 171 when(toICacheMeta.fire) { 172 XSDebug(p"PrefetchBuffer: write prefetch data to icache . vSetIdx:${meta_buffer(curr_move_ptr).vSetIdx} " + 173 p"paddr:${meta_buffer(curr_move_ptr).paddr}") 174 } 175 176 /** 177 ****************************************************************************** 178 * Prefetch buffer write logic 179 * Read old data when read and write occur in the same cycle 180 * There are 4 situations that can cause meta_buffer and data_buffer to be written 181 * - 1. ICacheMainPipe read hit that need to change some status 182 * - 2. Move prefetch cacheline to icache 183 * - 3. Receive write request from ICacheMissUnit 184 * - 4. Receive fencei req that need to flush all buffer 185 * Priority: 4 > 3 > 2 > 1 186 * The order of the code determines the priority, with the following priority being higher 187 ****************************************************************************** 188 */ 189 190 /** 1. confidence++ for every ICacheMainPipe hit */ 191 (0 until PortNumber) .map (i => 192 when(r_hit(i)) { 193 // handle overflow 194 when(meta_buffer(curr_hit_ptr(i)).confidence =/= maxIPFMoveConf.U) { 195 meta_buffer(curr_hit_ptr(i)).confidence := meta_buffer(curr_hit_ptr(i)).confidence + 1.U 196 } 197 } 198 ) 199 200 /** 2. set entry invalid after move to icache */ 201 when(toICacheMeta.fire) { 202 meta_buffer(curr_move_ptr).valid := false.B 203 } 204 205 /** 3. write prefetch data to entry */ 206 // try to get a free entry ptr 207 val free_oh = meta_buffer.map (_.valid === false.B) 208 val has_free = ParallelOR(free_oh) 209 val ptr_from_free = PriorityEncoder(free_oh) 210 // get a ptr from random replacement policy 211 val replacer = ReplacementPolicy.fromString(Some("random"), nPrefBufferEntries) 212 // write to free entry if has, otherwise random entry 213 val curr_write_ptr = Mux(has_free, ptr_from_free, replacer.way) 214 fromPIQ.ready := DontCare 215 when(fromPIQ.valid) { 216 meta_buffer(curr_write_ptr).valid := true.B 217 meta_buffer(curr_write_ptr).confidence := fromPIQ.bits.has_hit.asUInt 218 meta_buffer(curr_write_ptr).paddr := fromPIQ.bits.paddr 219 meta_buffer(curr_write_ptr).vSetIdx := fromPIQ.bits.vSetIdx 220 data_buffer(curr_write_ptr).cacheline := fromPIQ.bits.cacheline 221 } 222 223 if(DebugFlags.fdip) { 224 when(fromPIQ.valid) { 225 printf(p"PrefetchBuffer: receive prefetch data. vSetIdx:${fromPIQ.bits.vSetIdx} paddr:${fromPIQ.bits.paddr}") 226 } 227 } 228 229 XSPerfAccumulate("fdip_vitcm_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence =/= 0.U)) 230 XSPerfAccumulate("fdip_vitcm_no_used", !has_free && fromPIQ.valid && (meta_buffer(curr_write_ptr).confidence === 0.U)) 231 232 /** 4. flush all prefetch buffer when fencei */ 233 when(io.fencei) { 234 meta_buffer.foreach { b => 235 b.valid := false.B 236 b.confidence := 0.U 237 } 238 } 239 240 /** 241 ****************************************************************************** 242 * Register 2 cycle meta write info for IPrefetchPipe filter 243 ****************************************************************************** 244 */ 245 val meta_write_buffer = InitQueue(new FilterInfo, size = 2) 246 meta_write_buffer(0).valid := toICacheMeta.fire 247 meta_write_buffer(0).paddr := meta_buffer(curr_move_ptr).paddr 248 meta_write_buffer(1) := meta_write_buffer(0) 249 (0 until PortNumber).foreach (i => { 250 toIPrefetchInfo(i) := meta_write_buffer(i) 251 }) 252 253 XSPerfAccumulate("fdip_fencei_cycle", io.fencei) 254 255 if (env.EnableDifftest) { 256 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 257 difftest.coreid := io.hartId 258 difftest.index := 6.U 259 difftest.valid := toICacheData.fire 260 difftest.addr := toICacheData.bits.paddr 261 difftest.data := toICacheData.bits.data.asTypeOf(difftest.data) 262 difftest.idtfr := DontCare 263 } 264} 265 266class IPredfetchIO(implicit p: Parameters) extends IPrefetchBundle { 267 val ftqReq = Flipped(new FtqPrefechBundle) 268 val iTLBInter = new TlbRequestIO 269 val pmp = new ICachePMPBundle 270 val metaReadReq = Decoupled(new PrefetchMetaReadBundle) 271 val metaReadResp = Input(new PrefetchMetaRespBundle) 272 val prefetchReq = DecoupledIO(new PrefetchReq) 273 274 val IPFFilterRead = Flipped(new IPFBufferFilterRead) 275 val PIQFilterRead = Flipped(new PIQFilterRead) 276 277 val ipfRecentWrite = Input(Vec(2, new FilterInfo)) 278 val ICacheMissUnitInfo = Flipped(new ICacheMissUnitInfo) 279 val ICacheMainPipeInfo = Flipped(new ICacheMainPipeInfo) 280 281 val fencei = Input(Bool()) 282} 283 284class IPrefetchPipe(implicit p: Parameters) extends IPrefetchModule 285{ 286 val io = IO(new IPredfetchIO) 287 288 val enableBit = RegInit(false.B) 289 enableBit := enableICachePrefetch.B 290 291 val fromFtq = io.ftqReq 292 val (toITLB, fromITLB) = (io.iTLBInter.req, io.iTLBInter.resp) 293 val (toIMeta, fromIMeta, fromIMetaValid) = (io.metaReadReq, io.metaReadResp.metaData, io.metaReadResp.entryValid) 294 val (toIPFBuffer, fromIPFBuffer) = (io.IPFFilterRead.req, io.IPFFilterRead.resp) 295 val (toPIQ, fromPIQ) = (io.PIQFilterRead.req, io.PIQFilterRead.resp) 296 val (toPMP, fromPMP) = (io.pmp.req, io.pmp.resp) 297 val toPIQEnqReq = io.prefetchReq 298 299 val fromIPFInfo = io.ipfRecentWrite 300 val fromMainPipeInfo = io.ICacheMainPipeInfo 301 val fromMissUnitInfo = io.ICacheMissUnitInfo 302 303 val p0_fire, p1_fire, p2_fire = WireInit(false.B) 304 val p0_discard, p1_discard, p2_discard = WireInit(false.B) 305 val p0_ready, p1_ready, p2_ready = WireInit(false.B) 306 307 /** 308 ****************************************************************************** 309 * IPrefetch Stage 0 310 * - 1. send req to IMeta 311 * - 2. send req to ITLB (no blocked) 312 * - 3. check last req 313 ****************************************************************************** 314 */ 315 val p0_valid = fromFtq.req.valid 316 317 val p0_vaddr = addrAlign(fromFtq.req.bits.target, blockBytes, VAddrBits) 318 val p0_vaddr_reg = RegEnable(p0_vaddr, p0_fire) 319 val p0_req_cancel = Wire(Bool()) 320 321 /** 1. send req to IMeta */ 322 toIMeta.valid := p0_valid && !p0_req_cancel 323 toIMeta.bits.idx := get_idx(p0_vaddr) 324 325 /** 2. send req to ITLB (no blocked) */ 326 toITLB.valid := p0_valid && !p0_req_cancel 327 toITLB.bits.size := 3.U // TODO: fix the size 328 toITLB.bits.vaddr := p0_vaddr 329 toITLB.bits.debug.pc := p0_vaddr 330 toITLB.bits.kill := DontCare 331 toITLB.bits.cmd := TlbCmd.exec 332 toITLB.bits.debug.robIdx := DontCare 333 toITLB.bits.debug.isFirstIssue := DontCare 334 toITLB.bits.memidx := DontCare 335 toITLB.bits.no_translate := false.B 336 toITLB.bits.hyperinst := DontCare 337 toITLB.bits.hlvx := DontCare 338 fromITLB.ready := true.B 339 // TODO: whether to handle tlb miss for prefetch 340 io.iTLBInter.req_kill := true.B 341 342 /** FTQ request port */ 343 fromFtq.req.ready := p0_ready 344 345 /** 3. Check last req: request from FTQ is same as last time or behind */ 346 val p0_hit_behind = Wire(Bool()) 347 348 /** stage 0 control */ 349 // Cancel request when prefetch not enable or the request from FTQ is same as last time 350 p0_req_cancel := !enableBit || p0_hit_behind || io.fencei 351 p0_ready := p0_req_cancel || p1_ready && toITLB.ready && toIMeta.ready 352 p0_fire := p0_valid && p1_ready && toITLB.ready && toIMeta.ready && enableBit && !p0_req_cancel 353 p0_discard := p0_valid && p0_req_cancel 354 355 if(DebugFlags.fdip) { 356 when(p0_discard) { 357 printf(p"IPrefetchPipe: discard req in s0.") 358 } 359 } 360 361 /** 362 ****************************************************************************** 363 * IPrefetch Stage 1 364 * - 1. Receive resp from ITLB (no blocked) 365 * - 2. Receive resp from IMeta 366 * - 3. Check MSHR 367 ****************************************************************************** 368 */ 369 val p1_valid = generatePipeControl(lastFire = p0_fire, thisFire = p1_fire || p1_discard, thisFlush = false.B, lastFlush = false.B) 370 371 val p1_vaddr = RegEnable(p0_vaddr, 0.U(VAddrBits.W), p0_fire) 372 val p1_req_cancel = Wire(Bool()) 373 374 /** 1. Receive resp from ITLB (no blocked) */ 375 val p1_tlb_resp_miss = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.miss) 376 val p1_tlb_resp_paddr = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.paddr(0)) 377 val p1_tlb_resp_pf = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).pf.instr) 378 val p1_tlb_resp_af = ResultHoldBypass(valid = RegNext(p0_fire), data = fromITLB.bits.excp(0).af.instr) 379 val p1_exception = VecInit(Seq(p1_tlb_resp_pf, p1_tlb_resp_af)) 380 val p1_has_except = p1_exception.reduce(_ || _) 381 val p1_paddr = p1_tlb_resp_paddr 382 383 /** 2. Receive resp from IMeta. Register the data first because of strict timing. */ 384 val p1_meta_ptags_reg = RegEnable(VecInit(fromIMeta.map(way => way.tag)), RegNext(p0_fire)) 385 val p1_meta_valids_reg = RegEnable(fromIMetaValid, RegNext(p0_fire)) 386 387 /** Stage 1 control */ 388 p1_req_cancel := p1_tlb_resp_miss || p1_has_except || io.fencei 389 p1_ready := p1_valid && p2_ready || !p1_valid 390 p1_fire := p1_valid && !p1_req_cancel && p2_ready && enableBit 391 p1_discard := p1_valid && p1_req_cancel 392 393 when(p1_discard) { 394 XSDebug(p"IPrefetchPipe: discard req in s1. vaddr:${p1_vaddr}") 395 } 396 397 /** 398 ****************************************************************************** 399 * IPrefetch Stage 2 400 * - 1. IMeta Check: check whether req hit in icache 401 * - 2. PMP Check: send req and receive resq in the same cycle 402 * - 3. Prefetch buffer Check (if prefetch to L1) 403 * - 4. Prefetch Queue Check 404 * - 5. Prefetch Buffer Recently write chcek 405 * - 6. MainPipeInfo chcek 406 * - 7. MissUnitInfo chcek 407 * - 8. Recently meta write check 408 ****************************************************************************** 409 */ 410 val p2_valid = generatePipeControl(lastFire = p1_fire, thisFire = p2_fire || p2_discard, thisFlush = false.B, lastFlush = false.B) 411 412 val p2_paddr = RegEnable(p1_paddr, p1_fire) 413 val p2_vaddr = RegEnable(p1_vaddr, 0.U(VAddrBits.W), p1_fire) 414 val p2_req_cancel = Wire(Bool()) 415 val p2_vidx = get_idx(p2_vaddr) 416 417 p0_hit_behind := (p0_vaddr === p1_vaddr) || (p0_vaddr === p2_vaddr) 418 419 /** 1. IMeta Check */ 420 val p2_ptag = get_phy_tag(p2_paddr) 421 val p2_tag_eq_vec = VecInit(p1_meta_ptags_reg.map(_ === p2_ptag )) 422 val p2_tag_match_vec = VecInit(p2_tag_eq_vec.zipWithIndex.map{ case(way_tag_eq, w) => way_tag_eq && p1_meta_valids_reg(w)}) 423 val p2_tag_match = DataHoldBypass(ParallelOR(p2_tag_match_vec), RegNext(p1_fire)) 424 425 /** 2. PMP Check */ 426 // TODO: When fromPMP.mmio is low but fromPMP.instr is high, which is a except? 427 val p2_exception = DataHoldBypass((fromPMP.mmio && !fromPMP.instr) || fromPMP.instr, RegNext(p1_fire)) 428 // PMP request 429 toPMP.valid := p2_valid 430 toPMP.bits.addr := p2_paddr 431 toPMP.bits.size := 3.U 432 toPMP.bits.cmd := TlbCmd.exec 433 434 /** 3. Prefetch buffer Check */ 435 toIPFBuffer.paddr := p2_paddr 436 val p2_hit_buffer = fromIPFBuffer.ipf_hit 437 438 /** 4. Prefetch Queue Check */ 439 toPIQ.paddr := p2_paddr 440 val p2_hit_piq = fromPIQ.piq_hit 441 442 /** 5. Prefetch Buffer Recently write chcek */ 443 val p2_check_ipf_info = VecInit(fromIPFInfo.map(info => 444 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 445 446 /** 6. MainPipeInfo chcek */ 447 val check_mp_s1 = VecInit(fromMainPipeInfo.s1Info.map(info => 448 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 449 val check_mp_s2 = VecInit(fromMainPipeInfo.s2Info.map(info => 450 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 451 val check_mp_missSlot = VecInit(fromMainPipeInfo.missSlot.map(info => 452 info.valid && info.ptag === get_phy_tag(p2_paddr) && info.vSetIdx === p2_vidx)).reduce(_||_) 453 val p2_check_mp_info = check_mp_s1 || check_mp_s2 || check_mp_missSlot 454 455 /** 7. MissUnitInfo chcek */ 456 val check_mu_mshr = VecInit(fromMissUnitInfo.mshr.map(info => 457 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 458 val check_mu_recent_write = VecInit(fromMissUnitInfo.recentWrite.map(info => 459 info.valid && getBlkAddr(info.paddr) === getBlkAddr(p2_paddr))).reduce(_||_) 460 val p2_check_mu_info = check_mu_mshr || check_mu_recent_write 461 462 /** 8. send req to piq */ 463 toPIQEnqReq.valid := p2_valid && !p2_req_cancel 464 toPIQEnqReq.bits.paddr := p2_paddr 465 toPIQEnqReq.bits.vSetIdx := p2_vidx 466 467 /** Stage 2 control */ 468 p2_req_cancel := p2_tag_match || p2_exception || p2_hit_buffer || p2_hit_piq || 469 p2_check_ipf_info || p2_check_mp_info || p2_check_mu_info || io.fencei 470 p2_ready := p2_valid && toPIQEnqReq.ready || !p2_valid 471 p2_fire := toPIQEnqReq.fire 472 p2_discard := p2_valid && p2_req_cancel 473 474 when(p2_discard) { 475 XSDebug(p"IPrefetchPipe: discard req in s2. vaddr:${p2_vaddr}") 476 } 477 478 when(p2_fire) { 479 XSDebug(p"IPrefetchPipe: send req to PIQ. vaddr:${p2_vaddr} paddr:${p2_paddr}") 480 } 481 482 /** PerfAccumulate */ 483 // the number of prefetch request received from ftq 484 XSPerfAccumulate("prefetch_req_receive", fromFtq.req.fire) 485 // the number of prefetch request sent to PIQ 486 XSPerfAccumulate("prefetch_req_send_piq", toPIQEnqReq.fire) 487 /** 488 * Count the number of requests that are filtered for various reasons. 489 * The number of prefetch discard in Performance Accumulator may be 490 * a littel larger the number of really discarded. Because there can 491 * be multiple reasons for a canceled request at the same time. 492 */ 493 // discard prefetch request by fencei instruction 494 XSPerfAccumulate("fdip_prefetch_discard_by_fencei", (p0_discard || p1_discard || p2_discard) && io.fencei) 495 496 // discard prefetch request by the request is same as pipeline behind 497 XSPerfAccumulate("fdip_prefetch_discard_by_behind", p0_discard && p0_hit_behind) 498 // discard prefetch request by tlb miss 499 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_miss", p1_discard && p1_tlb_resp_miss) 500 // discard prefetch request by tlb except 501 XSPerfAccumulate("fdip_prefetch_discard_by_tlb_except", p1_discard && p1_has_except) 502 // discard prefetch request by hit mainPipe info 503 XSPerfAccumulate("fdip_prefetch_discard_by_mainPipe", p2_discard && p2_check_mp_info) 504 // discard prefetch request by hit missUnit info 505 XSPerfAccumulate("fdip_prefetch_discard_by_missUnit", p2_discard && p2_check_mu_info) 506 // discard prefetch request by hit icache 507 XSPerfAccumulate("fdip_prefetch_discard_by_hit_cache", p2_discard && p2_tag_match) 508 // discard prefetch request by pmp except or mmio 509 XSPerfAccumulate("fdip_prefetch_discard_by_pmp", p2_discard && p2_exception) 510 // discard prefetch request by hit in prefetch buffer 511 XSPerfAccumulate("fdip_prefetch_discard_by_hit_pf", p2_discard && p2_hit_buffer) 512 // discard prefetch request by hit in prefetch queue 513 XSPerfAccumulate("fdip_prefetch_discard_by_hit_piq", p2_discard && p2_hit_piq) 514} 515 516class PiqPtr(implicit p: Parameters) extends CircularQueuePtr[PiqPtr]( 517 p => p(XSCoreParamsKey).icacheParameters.nPrefetchEntries 518){ 519} 520 521object PiqPtr { 522 def apply(f: Bool, v: UInt)(implicit p: Parameters): PiqPtr = { 523 val ptr = Wire(new PiqPtr) 524 ptr.flag := f 525 ptr.value := v 526 ptr 527 } 528 def inverse(ptr: PiqPtr)(implicit p: Parameters): PiqPtr = { 529 apply(!ptr.flag, ptr.value) 530 } 531} 532 533class PrefetchQueueIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 534 val hartId = Input(UInt(8.W)) 535 val prefetchReq = Flipped(DecoupledIO(new PrefetchReq)) 536 val PIQFilterRead = new PIQFilterRead 537 val PIQRead = new PIQRead 538 val IPFBufferWrite = DecoupledIO(new IPFBufferWrite) 539 /** TileLink Port */ 540 val mem_acquire = DecoupledIO(new TLBundleA(edge.bundle)) 541 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 542 val fencei = Input(Bool()) 543} 544 545class PrefetchQueue(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule with HasCircularQueuePtrHelper { 546 val io = IO(new PrefetchQueueIO(edge)) 547 548 val enqueReq = io.prefetchReq 549 val fromIPrefetch = io.PIQFilterRead.req 550 val toIPrefetch = io.PIQFilterRead.resp 551 val fromMainPipe = io.PIQRead.req 552 val toMainPipe = io.PIQRead.resp 553 val toIPF = io.IPFBufferWrite 554 555 /** Define entryt content and initial FIFO queue */ 556 class PrefetchEntry(implicit p: Parameters) extends IPrefetchBundle { 557 val valid = Bool() 558 val paddr = UInt(PAddrBits.W) 559 val vSetIdx = UInt(idxBits.W) 560 } 561 val PIQ = RegInit(VecInit(Seq.fill(nPrefetchEntries)(0.U.asTypeOf((new PrefetchEntry).cloneType)))) 562 563 val enqPtr = RegInit(PiqPtr(false.B, 0.U)) 564 val deqPtr = RegInit(PiqPtr(false.B, 0.U)) 565 566 class HandleEntry(implicit p: Parameters) extends IPrefetchBundle { 567 val valid = Bool() 568 val paddr = UInt(PAddrBits.W) 569 val vSetIdx = UInt(idxBits.W) 570 val issue = Bool() 571 val finish = Bool() 572 val flash = Bool() 573 val readBeatCnt = UInt(log2Up(refillCycles).W) 574 val respData = Vec(refillCycles, UInt(beatBits.W)) 575 576 def cacheline = respData.asUInt 577 } 578 val handleEntry = RegInit(0.U.asTypeOf((new HandleEntry).cloneType)) 579 580 /** 581 ****************************************************************************** 582 * handle look-up request from IPrefetchPipe 583 * - 1. Receive request from IPrefetch 584 * - 2. Look up whether hit in PIQ and handleEntry 585 * - 3. Send response to IPrefetch at the same cycle 586 ****************************************************************************** 587 */ 588 val fr_block = get_block(fromIPrefetch.paddr) 589 val fr_hit_oh = PIQ.map(e => e.valid && (get_block(e.paddr) === fr_block)) :+ 590 (handleEntry.valid && (get_block(handleEntry.paddr) === fr_block)) 591 val fr_hit = ParallelOR(fr_hit_oh) 592 toIPrefetch.piq_hit := fr_hit 593 // only hit one entry in PIQ 594 assert(PopCount(fr_hit_oh) <= 1.U, "More than 1 hit in L1 PIQ for filter") 595 596 597 /** 598 ****************************************************************************** 599 * handle read request from ICacheMainPipe 600 * - 1. Receive request from ICacheMainPipe 601 * - 2. Look up PIQ and handleEntry 602 * - 3. Send response to ICacheMainPipe at the same cycle 603 ****************************************************************************** 604 */ 605 fromMainPipe.foreach(_.ready := DontCare) 606 val r_blocks = fromMainPipe.map (e => get_block(e.bits.paddr)) 607 /** When hit in PIQ, invalid hitted entry */ 608 val r_hit_oh = r_blocks.map (block => PIQ.map(e => e.valid && (get_block(e.paddr) === block))) 609 val curr_hit_ptr = r_hit_oh.map (OHToUInt(_)) 610 val r_hit_PIQ = (0 until PortNumber).map (i => ParallelOR(r_hit_oh(i)) && fromMainPipe(i).valid) 611 // only hit one entry in PIQ 612 (0 until PortNumber).map (i => 613 assert(PopCount(r_hit_oh(i)) <= 1.U, "More than 1 hit in L1 PIQ") 614 ) 615 616 /** When hit in handleEntry has't been issued, cancel it and return not hit at the same cycle. 617 * When hit in handleEntry has been issued return hit at the same cycle, then return data when transiton finish. 618 * ICacheMainPipe should be blocked during handle handleEntry. 619 */ 620 val r_hit_handle = r_blocks.map(block => handleEntry.valid && (block === get_block(handleEntry.paddr))) 621 val cancelHandleEntry = (((0 until PortNumber).map(i=> fromMainPipe(i).valid && r_hit_handle(i))).reduce(_||_) || io.fencei) && !handleEntry.issue 622 val piq_hit = (0 until PortNumber).map(i => r_hit_handle(i) && handleEntry.issue && !handleEntry.flash && !io.fencei) 623 toMainPipe := DontCare 624 if (prefetchToL1) { 625 (0 until PortNumber).foreach (i => { 626 toMainPipe(i).piq_hit := piq_hit(i) 627 toMainPipe(i).cacheline := handleEntry.cacheline 628 toMainPipe(i).data_valid := handleEntry.valid && handleEntry.finish 629 }) 630 } 631 val piq_hit_req = RegEnable(piq_hit.reduce(_||_), fromMainPipe.map(_.valid).reduce(_||_)) 632 633 XSPerfAccumulate("cancel_req_by_mainpipe_hit_handle", cancelHandleEntry && !io.mem_acquire.fire) 634 635 /** 636 ****************************************************************************** 637 * Manage prefetch requests by queue 638 * - 1. Free: invalid the entry hitted mainpipe 639 * - 2. Dequeue: move req to handle entry when handle entry is free 640 * - 3. Enqueue: enqueue new req, flush earlier req when PIQ is full 641 * - 4. Fencei: invalid all PIQ entry when fencei 642 * - 5. Dequeue: move prefetch request from last entry to handle entry 643 * - 6. Cancel handleEntry hitted mainpipe if hasn't been issued 644 * - 7. Valid flash of handleEntry when fencei if has been issued 645 ****************************************************************************** 646 */ 647 /** 1. Free: invalid the entry hitted mainpipe */ 648 (0 until PortNumber) .foreach (i => { 649 when(r_hit_PIQ(i)) { 650 PIQ(curr_hit_ptr(i)).valid := false.B 651 } 652 XSPerfAccumulate("cancel_req_by_mainpipe_hit_" + i.toString(), r_hit_PIQ(i)) 653 }) 654 655 /** 2. Dequeue: move req to handle entry when handle entry is free */ 656 // cancel entry when dequene req is hitted by mainpipe 657 val cancel = (0 until PortNumber).map (i => r_hit_PIQ(i) && (curr_hit_ptr(i) === deqPtr.value)).reduce(_||_) 658 when(!handleEntry.valid && !cancel && !isEmpty(enqPtr, deqPtr)) { 659 handleEntry.valid := PIQ(deqPtr.value).valid 660 handleEntry.paddr := PIQ(deqPtr.value).paddr 661 handleEntry.vSetIdx := PIQ(deqPtr.value).vSetIdx 662 handleEntry.issue := false.B 663 handleEntry.finish := false.B 664 handleEntry.flash := false.B 665 handleEntry.readBeatCnt := 0.U 666 PIQ(deqPtr.value).valid := false.B 667 deqPtr := deqPtr + 1.U 668 } 669 670 /** 3. Enqueue: enqueue new req, flush earlier req when PIQ is full */ 671 enqueReq.ready := true.B 672 when(enqueReq.valid) { 673 PIQ(enqPtr.value).valid := true.B 674 PIQ(enqPtr.value).vSetIdx := enqueReq.bits.vSetIdx 675 PIQ(enqPtr.value).paddr := enqueReq.bits.paddr 676 enqPtr := enqPtr + 1.U 677 when(isFull(enqPtr, deqPtr)) { 678 deqPtr := deqPtr + 1.U 679 } 680 } 681 XSPerfAccumulate("cancel_req_by_piq_full", enqueReq.valid && PIQ(enqPtr.value).valid) 682 683 /** 4. Fencei: invalid all PIQ entry when fencei */ 684 when(io.fencei) { 685 PIQ.map ( 686 _.valid := false.B 687 ) 688 enqPtr.value := 0.U 689 enqPtr.flag := false.B 690 deqPtr.value := 0.U 691 deqPtr.flag := false.B 692 } 693 694 /** 5. Cancel handleEntry hitted mainpipe or receive fencei if hasn't been issued */ 695 when(cancelHandleEntry) { 696 handleEntry.valid := false.B 697 } 698 699 /** 6. Valid flash of handleEntry when fencei. Drop data from TileLink port */ 700 when(io.fencei) { 701 handleEntry.flash := true.B 702 } 703 704 // deqPtr <= enqPtr 705 // assert(deqPtr <= enqPtr, "deqPtr must be not in front of enqPtr!") 706 assert((deqPtr.flag ^ enqPtr.flag) && (deqPtr.value >= enqPtr.value) || (!(deqPtr.flag ^ enqPtr.flag)) && (deqPtr.value <= enqPtr.value), "deqPtr must be not in front of enqPtr!") 707 708 /** 709 ****************************************************************************** 710 * Write Prefetch Data to Prefetch Buffer 711 * - valid signal only hold one cycle 712 ****************************************************************************** 713 */ 714 toIPF.valid := handleEntry.valid && handleEntry.finish 715 toIPF.bits.paddr := handleEntry.paddr 716 toIPF.bits.vSetIdx := handleEntry.vSetIdx 717 toIPF.bits.cacheline := handleEntry.cacheline 718 toIPF.bits.has_hit := piq_hit_req 719 720 721 /** 722 ****************************************************************************** 723 * Tilelink Transition 724 * - 1. Send acquire to L2Cache, after which the request cannot be canceled (issue valid) 725 * - 2. Wait data response from Tilelink 726 * - 3. Get data, valid finish (only one cycle) and invalid issue 727 ****************************************************************************** 728 */ 729 val (_, _, refill_done, refill_address_inc) = edge.addr_inc(io.mem_grant) 730 731 val s_idle :: s_memReadReq :: s_memReadResp :: s_write_back :: Nil = Enum(4) 732 val state = RegInit(s_idle) 733 734 // acquire port 735 io.mem_acquire.bits := DontCare 736 if (prefetchToL1) { 737 io.mem_acquire.bits := edge.Get( 738 fromSource = PortNumber.U, 739 toAddress = Cat(handleEntry.paddr(PAddrBits - 1, log2Ceil(blockBytes)), 0.U(log2Ceil(blockBytes).W)), 740 lgSize = (log2Up(cacheParams.blockBytes)).U)._2 741 } else { 742 io.mem_acquire.bits := edge.Hint( 743 fromSource = PortNumber.U, 744 toAddress = addrAlign(handleEntry.paddr, blockBytes, PAddrBits), 745 lgSize = (log2Up(cacheParams.blockBytes)).U, 746 param = TLHints.PREFETCH_READ)._2 747 } 748 749 io.mem_acquire.bits.user.lift(PreferCacheKey).foreach(_ := true.B) 750 io.mem_acquire.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.L1InstPrefetch.id.U) 751 752 // grant port 753 io.mem_grant.ready := true.B 754 io.mem_acquire.valid := (state === s_memReadReq) 755 756 if (prefetchToL1) { 757 switch(state) { 758 is(s_idle) { 759 when(handleEntry.valid && !cancelHandleEntry) { 760 handleEntry.issue := true.B 761 state := s_memReadReq 762 } 763 } 764 } 765 is(s_memReadReq) { 766 state := Mux(io.mem_acquire.fire, s_memReadResp, s_memReadReq) 767 } 768 is(s_memReadResp) { 769 when (edge.hasData(io.mem_grant.bits) && io.mem_grant.fire) { 770 handleEntry.readBeatCnt := handleEntry.readBeatCnt + 1.U 771 handleEntry.respData(handleEntry.readBeatCnt) := io.mem_grant.bits.data 772 when (handleEntry.readBeatCnt === (refillCycles - 1).U) { 773 assert(refill_done, "refill not done!") 774 state := s_write_back 775 when (!io.fencei && !handleEntry.flash) { 776 handleEntry.issue := false.B 777 handleEntry.finish := true.B 778 } 779 } 780 } 781 is(s_write_back) { 782 state := s_idle 783 handleEntry.valid := false.B 784 } 785 } 786 } else { 787 switch(state) { 788 is(s_idle) { 789 when(handleEntry.valid && !cancelHandleEntry) { 790 state := s_memReadReq 791 } 792 } 793 is(s_memReadReq) { 794 when(io.mem_acquire.fire || cancelHandleEntry) { 795 state := s_idle 796 handleEntry.valid := false.B 797 } otherwise { 798 state := s_memReadReq 799 } 800 } 801 } 802 } 803 804 // the number of prefetch request sent to L2Cache 805 XSPerfAccumulate("prefetch_req_send_L2", io.mem_acquire.fire) 806 807 if (env.EnableDifftest) { 808 val diffipfrefill = DifftestModule(new DiffRefillEvent, dontCare = true) 809 diffipfrefill.coreid := io.hartId 810 diffipfrefill.index := 3.U 811 diffipfrefill.valid := handleEntry.valid && handleEntry.finish 812 diffipfrefill.addr := handleEntry.paddr 813 diffipfrefill.data := handleEntry.cacheline.asTypeOf(diffipfrefill.data) 814 diffipfrefill.idtfr := DontCare 815 } 816} 817 818class FDIPPrefetchIO(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchBundle { 819 /** commen */ 820 val fencei = Input(Bool()) 821 val hartId = Input(UInt(hartIdLen.W)) 822 823 /** Prefetch Mainpipe IO */ 824 val ftqReq = Flipped(new FtqPrefechBundle) 825 val iTLBInter = new TlbRequestIO 826 val pmp = new ICachePMPBundle 827 val metaReadReq = Decoupled(new PrefetchMetaReadBundle) 828 val metaReadResp = Input(new PrefetchMetaRespBundle) 829 830 val ICacheMissUnitInfo = Flipped(new ICacheMissUnitInfo) 831 val ICacheMainPipeInfo = Flipped(new ICacheMainPipeInfo) 832 833 /** Prefetch Buffer IO */ 834 val IPFBufferRead = new IPFBufferRead 835 val metaWrite = DecoupledIO(new ICacheMetaWriteBundle) 836 val dataWrite = DecoupledIO(new ICacheDataWriteBundle) 837 val IPFReplacer = new IPFReplacer 838 839 /** Prefetch Queue IO */ 840 val PIQRead = new PIQRead 841 val mem_acquire = new DecoupledIO(new TLBundleA(edge.bundle)) 842 val mem_grant = Flipped(DecoupledIO(new TLBundleD(edge.bundle))) 843} 844 845class FDIPPrefetch(edge: TLEdgeOut)(implicit p: Parameters) extends IPrefetchModule 846{ 847 val io = IO(new FDIPPrefetchIO(edge)) 848 849 val prefetchPipe = Module(new IPrefetchPipe) 850 val prefetchQueue = Module(new PrefetchQueue(edge)) 851 852 io.metaReadReq <> prefetchPipe.io.metaReadReq 853 io.metaWrite <> DontCare 854 io.dataWrite <> DontCare 855 io.IPFReplacer <> DontCare 856 io.IPFBufferRead <> DontCare 857 io.mem_acquire <> prefetchQueue.io.mem_acquire 858 io.mem_grant <> prefetchQueue.io.mem_grant 859 860 // outside 861 prefetchPipe.io.fencei <> io.fencei 862 prefetchPipe.io.ftqReq <> io.ftqReq 863 prefetchPipe.io.iTLBInter <> io.iTLBInter 864 prefetchPipe.io.pmp <> io.pmp 865 prefetchPipe.io.metaReadResp <> io.metaReadResp 866 prefetchPipe.io.ICacheMissUnitInfo <> io.ICacheMissUnitInfo 867 prefetchPipe.io.ICacheMainPipeInfo <> io.ICacheMainPipeInfo 868 // inside 869 prefetchPipe.io.ipfRecentWrite <> DontCare 870 prefetchPipe.io.IPFFilterRead <> DontCare 871 872 // outside 873 prefetchQueue.io.hartId <> io.hartId 874 prefetchQueue.io.fencei <> io.fencei 875 prefetchQueue.io.PIQRead <> io.PIQRead 876 // inside 877 prefetchQueue.io.prefetchReq <> prefetchPipe.io.prefetchReq 878 prefetchQueue.io.PIQFilterRead <> prefetchPipe.io.PIQFilterRead 879 prefetchQueue.io.IPFBufferWrite <> DontCare 880 881 if (prefetchToL1) { 882 val prefetchBuffer = Module(new PrefetchBuffer) 883 // outside 884 prefetchBuffer.io.hartId <> io.hartId 885 prefetchBuffer.io.fencei <> io.fencei 886 prefetchBuffer.io.IPFBufferRead <> io.IPFBufferRead 887 prefetchBuffer.io.metaWrite <> io.metaWrite 888 prefetchBuffer.io.dataWrite <> io.dataWrite 889 prefetchBuffer.io.IPFReplacer <> io.IPFReplacer 890 // inside 891 prefetchBuffer.io.IPFFilterRead <> prefetchPipe.io.IPFFilterRead 892 prefetchBuffer.io.ipfRecentWrite <> prefetchPipe.io.ipfRecentWrite 893 prefetchBuffer.io.IPFBufferWrite <> prefetchQueue.io.IPFBufferWrite 894 } 895}