1/*************************************************************************************** 2 * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3 * Copyright (c) 2020-2021 Peng Cheng Laboratory 4 * 5 * XiangShan is licensed under Mulan PSL v2. 6 * You can use this software according to the terms and conditions of the Mulan PSL v2. 7 * You may obtain a copy of Mulan PSL v2 at: 8 * http://license.coscl.org.cn/MulanPSL2 9 * 10 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13 * 14 * See the Mulan PSL v2 for more details. 15 ***************************************************************************************/ 16 17package xiangshan.frontend.icache 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import difftest._ 23import freechips.rocketchip.tilelink._ 24import utils._ 25import xiangshan.cache.mmu._ 26import xiangshan.frontend._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28import huancun.PreferCacheKey 29import xiangshan.XSCoreParamsKey 30import utility._ 31 32abstract class IPrefetchBundle(implicit p: Parameters) extends ICacheBundle 33abstract class IPrefetchModule(implicit p: Parameters) extends ICacheModule 34 35class IPredfetchIO(implicit p: Parameters) extends IPrefetchBundle { 36 // control 37 val csr_pf_enable = Input(Bool()) 38 val flush = Input(Bool()) 39 40 val ftqReq = Flipped(new FtqToPrefetchIO) 41 val itlb = Vec(PortNumber, new TlbRequestIO) 42 val pmp = Vec(PortNumber, new ICachePMPBundle) 43 val metaRead = new ICacheMetaReqBundle 44 val MSHRReq = DecoupledIO(new ICacheMissReq) 45 val MSHRResp = Flipped(ValidIO(new ICacheMissResp)) 46 val wayLookupWrite = DecoupledIO(new WayLookupInfo) 47} 48 49class IPrefetchPipe(implicit p: Parameters) extends IPrefetchModule 50{ 51 val io = IO(new IPredfetchIO) 52 53 val fromFtq = io.ftqReq 54 val (toITLB, fromITLB) = (io.itlb.map(_.req), io.itlb.map(_.resp)) 55 val (toPMP, fromPMP) = (io.pmp.map(_.req), io.pmp.map(_.resp)) 56 val (toMeta, fromMeta) = (io.metaRead.toIMeta, io.metaRead.fromIMeta) 57 val (toMSHR, fromMSHR) = (io.MSHRReq, io.MSHRResp) 58 val toWayLookup = io.wayLookupWrite 59 60 val enableBit = RegInit(false.B) 61 enableBit := io.csr_pf_enable 62 63 val s0_fire, s1_fire, s2_fire = WireInit(false.B) 64 val s0_discard, s2_discard = WireInit(false.B) 65 val s0_ready, s1_ready, s2_ready = WireInit(false.B) 66 val s0_flush, s1_flush, s2_flush = WireInit(false.B) 67 val from_bpu_s0_flush, from_bpu_s1_flush = WireInit(false.B) 68 69 /** 70 ****************************************************************************** 71 * IPrefetch Stage 0 72 * - 1. receive ftq req 73 * - 2. send req to ITLB 74 * - 3. send req to Meta SRAM 75 ****************************************************************************** 76 */ 77 val s0_valid = fromFtq.req.valid 78 79 /** 80 ****************************************************************************** 81 * receive ftq req 82 ****************************************************************************** 83 */ 84 val s0_req_vaddr = VecInit(Seq(fromFtq.req.bits.startAddr, fromFtq.req.bits.nextlineStart)) 85 val s0_req_ftqIdx = fromFtq.req.bits.ftqIdx 86 val s0_doubleline = fromFtq.req.bits.crossCacheline 87 val s0_req_vSetIdx = s0_req_vaddr.map(get_idx(_)) 88 89 from_bpu_s0_flush := fromFtq.flushFromBpu.shouldFlushByStage2(s0_req_ftqIdx) || 90 fromFtq.flushFromBpu.shouldFlushByStage3(s0_req_ftqIdx) 91 s0_flush := io.flush || from_bpu_s0_flush || s1_flush 92 93 val s0_can_go = s1_ready && toITLB(0).ready && toITLB(1).ready && toMeta.ready 94 fromFtq.req.ready := s0_can_go 95 96 s0_fire := s0_valid && s0_can_go && !s0_flush 97 98 /** 99 ****************************************************************************** 100 * IPrefetch Stage 1 101 * - 1. Receive resp from ITLB 102 * - 2. Receive resp from IMeta and check 103 * - 3. Monitor the requests from missUnit to write to SRAM. 104 * - 4. Wirte wayLookup 105 ****************************************************************************** 106 */ 107 val s1_valid = generatePipeControl(lastFire = s0_fire, thisFire = s1_fire, thisFlush = s1_flush, lastFlush = false.B) 108 109 val s1_req_vaddr = RegEnable(s0_req_vaddr, 0.U.asTypeOf(s0_req_vaddr), s0_fire) 110 val s1_doubleline = RegEnable(s0_doubleline, 0.U.asTypeOf(s0_doubleline), s0_fire) 111 val s1_req_ftqIdx = RegEnable(s0_req_ftqIdx, 0.U.asTypeOf(s0_req_ftqIdx), s0_fire) 112 val s1_req_vSetIdx = VecInit(s1_req_vaddr.map(get_idx(_))) 113 114 val m_idle :: m_itlbResend :: m_metaResend :: m_enqWay :: m_enterS2 :: Nil = Enum(5) 115 val state = RegInit(m_idle) 116 val next_state = WireDefault(state) 117 val s0_fire_r = RegNext(s0_fire) 118 dontTouch(state) 119 dontTouch(next_state) 120 state := next_state 121 122 /** 123 ****************************************************************************** 124 * resend itlb req if miss 125 ****************************************************************************** 126 */ 127 val s1_wait_itlb = RegInit(VecInit(Seq.fill(PortNumber)(false.B))) 128 (0 until PortNumber).foreach { i => 129 when(s1_flush) { 130 s1_wait_itlb(i) := false.B 131 }.elsewhen(RegNext(s0_fire) && fromITLB(i).bits.miss) { 132 s1_wait_itlb(i) := true.B 133 }.elsewhen(s1_wait_itlb(i) && !fromITLB(i).bits.miss) { 134 s1_wait_itlb(i) := false.B 135 } 136 } 137 val s1_need_itlb = VecInit(Seq((RegNext(s0_fire) || s1_wait_itlb(0)) && fromITLB(0).bits.miss, 138 (RegNext(s0_fire) || s1_wait_itlb(1)) && fromITLB(1).bits.miss && s1_doubleline)) 139 val tlb_valid_pulse = VecInit(Seq((RegNext(s0_fire) || s1_wait_itlb(0)) && !fromITLB(0).bits.miss, 140 (RegNext(s0_fire) || s1_wait_itlb(1)) && !fromITLB(1).bits.miss && s1_doubleline)) 141 val tlb_valid_latch = VecInit((0 until PortNumber).map(i => ValidHoldBypass(tlb_valid_pulse(i), s1_fire, flush=s1_flush))) 142 val itlb_finish = tlb_valid_latch(0) && (!s1_doubleline || tlb_valid_latch(1)) 143 144 for (i <- 0 until PortNumber) { 145 toITLB(i).valid := s1_need_itlb(i) || (s0_valid && (if(i == 0) true.B else s0_doubleline)) 146 toITLB(i).bits := DontCare 147 toITLB(i).bits.size := 3.U 148 toITLB(i).bits.vaddr := Mux(s1_need_itlb(i), s1_req_vaddr(i), s0_req_vaddr(i)) 149 toITLB(i).bits.debug.pc := Mux(s1_need_itlb(i), s1_req_vaddr(i), s0_req_vaddr(i)) 150 toITLB(i).bits.cmd := TlbCmd.exec 151 toITLB(i).bits.no_translate := false.B 152 } 153 fromITLB.foreach(_.ready := true.B) 154 io.itlb.foreach(_.req_kill := false.B) 155 156 /** 157 ****************************************************************************** 158 * Receive resp from ITLB 159 ****************************************************************************** 160 */ 161 val s1_req_paddr_wire = VecInit(fromITLB.map(_.bits.paddr(0))) 162 val s1_req_paddr_reg = VecInit((0 until PortNumber).map(i => 163 RegEnable(s1_req_paddr_wire(i), 0.U(PAddrBits.W), tlb_valid_pulse(i)))) 164 val s1_req_paddr = VecInit((0 until PortNumber).map(i => 165 Mux(tlb_valid_pulse(i), s1_req_paddr_wire(i), s1_req_paddr_reg(i)))) 166 val s1_req_gpaddr_tmp = VecInit((0 until PortNumber).map(i => 167 ResultHoldBypass(valid = tlb_valid_pulse(i), init = 0.U.asTypeOf(fromITLB(i).bits.gpaddr(0)), data = fromITLB(i).bits.gpaddr(0)))) 168 val itlbExcpPF = VecInit((0 until PortNumber).map(i => 169 ResultHoldBypass(valid = tlb_valid_pulse(i), init = 0.U.asTypeOf(fromITLB(i).bits.excp(0).pf.instr), data = fromITLB(i).bits.excp(0).pf.instr))) 170 val itlbExcpGPF = VecInit((0 until PortNumber).map(i => 171 ResultHoldBypass(valid = tlb_valid_pulse(i), init = 0.U.asTypeOf(fromITLB(i).bits.excp(0).gpf.instr), data = fromITLB(i).bits.excp(0).gpf.instr))) 172 val itlbExcpAF = VecInit((0 until PortNumber).map(i => 173 ResultHoldBypass(valid = tlb_valid_pulse(i), init = 0.U.asTypeOf(fromITLB(i).bits.excp(0).af.instr), data = fromITLB(i).bits.excp(0).af.instr))) 174 val itlbExcp = VecInit((0 until PortNumber).map(i => itlbExcpAF(i) || itlbExcpPF(i) || itlbExcpGPF(i))) 175 176 /* Select gpaddr with the first gpf 177 * Note: the backend wants the base guest physical address of a fetch block 178 * for port(i), its base gpaddr is actually (gpaddr - i * blocksize) 179 * see GPAMem: https://github.com/OpenXiangShan/XiangShan/blob/344cf5d55568dd40cd658a9ee66047a505eeb504/src/main/scala/xiangshan/backend/GPAMem.scala#L33-L34 180 * see also: https://github.com/OpenXiangShan/XiangShan/blob/344cf5d55568dd40cd658a9ee66047a505eeb504/src/main/scala/xiangshan/frontend/IFU.scala#L374-L375 181 */ 182 val s1_req_gpaddr = PriorityMuxDefault( 183 itlbExcpGPF zip (0 until PortNumber).map(i => s1_req_gpaddr_tmp(i) - (i << blockOffBits).U), 184 0.U.asTypeOf(s1_req_gpaddr_tmp(0)) 185 ) 186 187 /** 188 ****************************************************************************** 189 * resend metaArray read req when itlb miss finish 190 ****************************************************************************** 191 */ 192 val s1_need_meta = ((state === m_itlbResend) && itlb_finish) || (state === m_metaResend) 193 toMeta.valid := s1_need_meta || s0_valid 194 toMeta.bits := DontCare 195 toMeta.bits.isDoubleLine := Mux(s1_need_meta, s1_doubleline, s0_doubleline) 196 197 for (i <- 0 until PortNumber) { 198 toMeta.bits.vSetIdx(i) := Mux(s1_need_meta, s1_req_vSetIdx(i), s0_req_vSetIdx(i)) 199 } 200 201 /** 202 ****************************************************************************** 203 * Receive resp from IMeta and check 204 ****************************************************************************** 205 */ 206 val s1_req_ptags = VecInit(s1_req_paddr.map(get_phy_tag(_))) 207 208 val s1_meta_ptags = fromMeta.tags 209 val s1_meta_valids = fromMeta.entryValid 210 val s1_meta_errors = VecInit((0 until PortNumber).map( p => 211 // If error is found in either way, the tag_eq_vec is unreliable, so we do not use waymask, but directly .orR 212 fromMeta.errors(p).asUInt.orR 213 )) 214 215 def get_waymask(paddrs: Vec[UInt]): Vec[UInt] = { 216 val ptags = paddrs.map(get_phy_tag(_)) 217 val tag_eq_vec = VecInit((0 until PortNumber).map( p => VecInit((0 until nWays).map( w => s1_meta_ptags(p)(w) === ptags(p))))) 218 val tag_match_vec = VecInit((0 until PortNumber).map( k => VecInit(tag_eq_vec(k).zipWithIndex.map{ case(way_tag_eq, w) => way_tag_eq && s1_meta_valids(k)(w)}))) 219 val waymasks = VecInit(tag_match_vec.map(_.asUInt)) 220 waymasks 221 } 222 223 val s1_SRAM_waymasks = VecInit((0 until PortNumber).map(i => 224 Mux(tlb_valid_pulse(i), get_waymask(s1_req_paddr_wire)(i), get_waymask(s1_req_paddr_reg)(i)))) 225 226 /** 227 ****************************************************************************** 228 * update waymask according to MSHR update data 229 ****************************************************************************** 230 */ 231 def update_waymask(mask: UInt, vSetIdx: UInt, ptag: UInt): UInt = { 232 require(mask.getWidth == nWays) 233 val new_mask = WireInit(mask) 234 val valid = fromMSHR.valid && !fromMSHR.bits.corrupt 235 val vset_same = fromMSHR.bits.vSetIdx === vSetIdx 236 val ptag_same = getPhyTagFromBlk(fromMSHR.bits.blkPaddr) === ptag 237 val way_same = fromMSHR.bits.waymask === mask 238 when(valid && vset_same) { 239 when(ptag_same) { 240 new_mask := fromMSHR.bits.waymask 241 }.elsewhen(way_same) { 242 new_mask := 0.U 243 } 244 } 245 new_mask 246 } 247 248 val s1_SRAM_valid = s0_fire_r || RegNext(s1_need_meta && toMeta.ready) 249 val s1_MSHR_valid = fromMSHR.valid && !fromMSHR.bits.corrupt 250 val s1_waymasks = WireInit(VecInit(Seq.fill(PortNumber)(0.U(nWays.W)))) 251 val s1_waymasks_r = RegEnable(s1_waymasks, 0.U.asTypeOf(s1_waymasks), s1_SRAM_valid || s1_MSHR_valid) 252 (0 until PortNumber).foreach{i => 253 val old_waymask = Mux(s1_SRAM_valid, s1_SRAM_waymasks(i), s1_waymasks_r(i)) 254 s1_waymasks(i) := update_waymask(old_waymask, s1_req_vSetIdx(i), s1_req_ptags(i)) 255 } 256 257 /** 258 ****************************************************************************** 259 * send enqueu req to WayLookup 260 ******** ********************************************************************** 261 */ 262 // Disallow enqueuing wayLookup when SRAM write occurs. 263 toWayLookup.valid := ((state === m_enqWay) || ((state === m_idle) && itlb_finish)) && !s1_flush && !fromMSHR.valid 264 toWayLookup.bits.vSetIdx := s1_req_vSetIdx 265 toWayLookup.bits.waymask := s1_waymasks 266 toWayLookup.bits.ptag := s1_req_ptags 267 toWayLookup.bits.gpaddr := s1_req_gpaddr 268 (0 until PortNumber).foreach { i => 269 val excpValid = (if (i == 0) true.B else s1_doubleline) // exception in first line is always valid, in second line is valid iff is doubleline request 270 toWayLookup.bits.excp_tlb_af(i) := excpValid && itlbExcpAF(i) 271 toWayLookup.bits.excp_tlb_pf(i) := excpValid && itlbExcpPF(i) 272 toWayLookup.bits.excp_tlb_gpf(i) := excpValid && itlbExcpGPF(i) 273 toWayLookup.bits.meta_errors(i) := excpValid && s1_meta_errors(i) 274 } 275 276 val s1_waymasks_vec = s1_waymasks.map(_.asTypeOf(Vec(nWays, Bool()))) 277 when(toWayLookup.fire) { 278 assert(PopCount(s1_waymasks_vec(0)) <= 1.U && (PopCount(s1_waymasks_vec(1)) <= 1.U || !s1_doubleline), 279 "Multiple hit in main pipe, port0:is=%d,ptag=0x%x,vidx=0x%x,vaddr=0x%x port1:is=%d,ptag=0x%x,vidx=0x%x,vaddr=0x%x ", 280 PopCount(s1_waymasks_vec(0)) > 1.U, s1_req_ptags(0), get_idx(s1_req_vaddr(0)), s1_req_vaddr(0), 281 PopCount(s1_waymasks_vec(1)) > 1.U && s1_doubleline, s1_req_ptags(1), get_idx(s1_req_vaddr(1)), s1_req_vaddr(1)) 282 } 283 284 /** 285 ****************************************************************************** 286 * PMP check 287 ****************************************************************************** 288 */ 289 toPMP.zipWithIndex.map { case (p, i) => 290 p.valid := s1_valid 291 p.bits.addr := s1_req_paddr(i) 292 p.bits.size := 3.U // TODO 293 p.bits.cmd := TlbCmd.exec 294 } 295 val pmpExcp = VecInit(Seq(fromPMP(0).instr || fromPMP(0).mmio, 296 fromPMP(0).instr || fromPMP(1).instr || fromPMP(0).mmio)) 297 298 /** 299 ****************************************************************************** 300 * state machine 301 ******** ********************************************************************** 302 */ 303 304 switch(state) { 305 is(m_idle) { 306 when(s1_valid && !itlb_finish) { 307 next_state := m_itlbResend 308 }.elsewhen(s1_valid && itlb_finish && !toWayLookup.fire) { 309 next_state := m_enqWay 310 }.elsewhen(s1_valid && itlb_finish && toWayLookup.fire && !s2_ready) { 311 next_state := m_enterS2 312 } 313 } 314 is(m_itlbResend) { 315 when(itlb_finish && !toMeta.ready) { 316 next_state := m_metaResend 317 }.elsewhen(itlb_finish && toMeta.ready) { 318 next_state := m_enqWay 319 } 320 } 321 is(m_metaResend) { 322 when(toMeta.ready) { 323 next_state := m_enqWay 324 } 325 } 326 is(m_enqWay) { 327 when(toWayLookup.fire && !s2_ready) { 328 next_state := m_enterS2 329 }.elsewhen(toWayLookup.fire && s2_ready) { 330 next_state := m_idle 331 } 332 } 333 is(m_enterS2) { 334 when(s2_ready) { 335 next_state := m_idle 336 } 337 } 338 } 339 340 when(s1_flush) { 341 next_state := m_idle 342 } 343 344 /** Stage 1 control */ 345 from_bpu_s1_flush := s1_valid && fromFtq.flushFromBpu.shouldFlushByStage3(s1_req_ftqIdx) 346 s1_flush := io.flush || from_bpu_s1_flush 347 348 s1_ready := next_state === m_idle 349 s1_fire := (next_state === m_idle) && s1_valid && !s1_flush 350 351 /** 352 ****************************************************************************** 353 * IPrefetch Stage 2 354 * - 1. Monitor the requests from missUnit to write to SRAM. 355 * - 2. send req to missUnit 356 ****************************************************************************** 357 */ 358 val s2_valid = generatePipeControl(lastFire = s1_fire, thisFire = s2_fire, thisFlush = s2_flush, lastFlush = false.B) 359 360 val s2_req_vaddr = RegEnable(s1_req_vaddr, 0.U.asTypeOf(s1_req_vaddr), s1_fire) 361 val s2_doubleline = RegEnable(s1_doubleline, 0.U.asTypeOf(s1_doubleline), s1_fire) 362 val s2_req_paddr = RegEnable(s1_req_paddr, 0.U.asTypeOf(s1_req_paddr), s1_fire) 363 364 val s2_pmpExcp = RegEnable(pmpExcp, 0.U.asTypeOf(pmpExcp), s1_fire) 365 val s2_itlbExcp = RegEnable(itlbExcp, 0.U.asTypeOf(itlbExcp), s1_fire) 366 val s2_waymasks = RegEnable(s1_waymasks, 0.U.asTypeOf(s1_waymasks), s1_fire) 367 368 val s2_req_vSetIdx = s2_req_vaddr.map(get_idx(_)) 369 val s2_req_ptags = s2_req_paddr.map(get_phy_tag(_)) 370 371 /** 372 ****************************************************************************** 373 * Monitor the requests from missUnit to write to SRAM 374 ****************************************************************************** 375 */ 376 val s2_MSHR_match = VecInit((0 until PortNumber).map(i => (s2_req_vSetIdx(i) === fromMSHR.bits.vSetIdx) && 377 (s2_req_ptags(i) === getPhyTagFromBlk(fromMSHR.bits.blkPaddr)) && 378 s2_valid && fromMSHR.valid && !fromMSHR.bits.corrupt)) 379 val s2_MSHR_hits = (0 until PortNumber).map(i => ValidHoldBypass(s2_MSHR_match(i), s2_fire || s2_flush)) 380 381 val s2_hits = s2_waymasks.map(_.orR) 382 val s2_miss = VecInit(Seq(!s2_itlbExcp(0) && !s2_pmpExcp(0) && !s2_hits(0) && !s2_MSHR_hits(0), 383 !s2_itlbExcp(0) && !s2_pmpExcp(0) && !s2_itlbExcp(1) && !s2_pmpExcp(1) && !s2_hits(1) && !s2_MSHR_hits(1) && s2_doubleline)) 384 385 /** 386 ****************************************************************************** 387 * send req to missUnit 388 ****************************************************************************** 389 */ 390 val toMSHRArbiter = Module(new Arbiter(new ICacheMissReq, PortNumber)) 391 392 // To avoid sending duplicate requests. 393 val has_send = RegInit(VecInit(Seq.fill(2)(false.B))) 394 (0 until PortNumber).foreach{ i => 395 when(s1_fire) { 396 has_send(i) := false.B 397 }.elsewhen(toMSHRArbiter.io.in(i).fire) { 398 has_send(i) := true.B 399 } 400 } 401 402 (0 until PortNumber).map{ i => 403 toMSHRArbiter.io.in(i).valid := s2_valid && s2_miss(i) && !has_send(i) 404 toMSHRArbiter.io.in(i).bits.blkPaddr := getBlkAddr(s2_req_paddr(i)) 405 toMSHRArbiter.io.in(i).bits.vSetIdx := s2_req_vSetIdx(i) 406 } 407 408 toMSHR <> toMSHRArbiter.io.out 409 410 s2_flush := io.flush 411 412 val s2_finish = (0 until PortNumber).map(i => has_send(i) || !s2_miss(i) || toMSHRArbiter.io.in(i).fire).reduce(_&&_) 413 s2_ready := s2_finish || !s2_valid 414 s2_fire := s2_valid && s2_finish && !s2_flush 415 416 /** PerfAccumulate */ 417 // the number of prefetch request received from ftq 418 XSPerfAccumulate("prefetch_req_receive", fromFtq.req.fire) 419 // the number of prefetch request sent to missUnit 420 XSPerfAccumulate("prefetch_req_send", toMSHR.fire) 421 XSPerfAccumulate("to_missUnit_stall", toMSHR.valid && !toMSHR.ready) 422 /** 423 * Count the number of requests that are filtered for various reasons. 424 * The number of prefetch discard in Performance Accumulator may be 425 * a littel larger the number of really discarded. Because there can 426 * be multiple reasons for a canceled request at the same time. 427 */ 428 // discard prefetch request by flush 429 // XSPerfAccumulate("fdip_prefetch_discard_by_tlb_except", p1_discard && p1_tlb_except) 430 // // discard prefetch request by hit icache SRAM 431 // XSPerfAccumulate("fdip_prefetch_discard_by_hit_cache", p2_discard && p1_meta_hit) 432 // // discard prefetch request by hit wirte SRAM 433 // XSPerfAccumulate("fdip_prefetch_discard_by_p1_monoitor", p1_discard && p1_monitor_hit) 434 // // discard prefetch request by pmp except or mmio 435 // XSPerfAccumulate("fdip_prefetch_discard_by_pmp", p2_discard && p2_pmp_except) 436 // // discard prefetch request by hit mainPipe info 437 // // XSPerfAccumulate("fdip_prefetch_discard_by_mainPipe", p2_discard && p2_mainPipe_hit) 438}