1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import freechips.rocketchip.rocket.RVCDecoder 23import xiangshan._ 24import xiangshan.cache.mmu._ 25import xiangshan.frontend.icache._ 26import utils._ 27import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 28 29trait HasInstrMMIOConst extends HasXSParameter with HasIFUConst{ 30 def mmioBusWidth = 64 31 def mmioBusBytes = mmioBusWidth / 8 32 def maxInstrLen = 32 33} 34 35trait HasIFUConst extends HasXSParameter{ 36 def addrAlign(addr: UInt, bytes: Int, highest: Int): UInt = Cat(addr(highest-1, log2Ceil(bytes)), 0.U(log2Ceil(bytes).W)) 37 def fetchQueueSize = 2 38 39 def getBasicBlockIdx( pc: UInt, start: UInt ): UInt = { 40 val byteOffset = pc - start 41 (byteOffset - instBytes.U)(log2Ceil(PredictWidth),instOffsetBits) 42 } 43} 44 45class IfuToFtqIO(implicit p:Parameters) extends XSBundle { 46 val pdWb = Valid(new PredecodeWritebackBundle) 47} 48 49class FtqInterface(implicit p: Parameters) extends XSBundle { 50 val fromFtq = Flipped(new FtqToIfuIO) 51 val toFtq = new IfuToFtqIO 52} 53 54class UncacheInterface(implicit p: Parameters) extends XSBundle { 55 val fromUncache = Flipped(DecoupledIO(new InsUncacheResp)) 56 val toUncache = DecoupledIO( new InsUncacheReq ) 57} 58class NewIFUIO(implicit p: Parameters) extends XSBundle { 59 val ftqInter = new FtqInterface 60 val icacheInter = Vec(2, Flipped(new ICacheMainPipeBundle)) 61 val icacheStop = Output(Bool()) 62 val icachePerfInfo = Input(new ICachePerfInfo) 63 val toIbuffer = Decoupled(new FetchToIBuffer) 64 val uncacheInter = new UncacheInterface 65 val frontendTrigger = Flipped(new FrontendTdataDistributeIO) 66 val csrTriggerEnable = Input(Vec(4, Bool())) 67 val rob_commits = Flipped(Vec(CommitWidth, Valid(new RobCommitInfo))) 68 val iTLBInter = new BlockTlbRequestIO 69 val pmp = new IPrefetchPMPBundle 70} 71 72// record the situation in which fallThruAddr falls into 73// the middle of an RVI inst 74class LastHalfInfo(implicit p: Parameters) extends XSBundle { 75 val valid = Bool() 76 val middlePC = UInt(VAddrBits.W) 77 def matchThisBlock(startAddr: UInt) = valid && middlePC === startAddr 78} 79 80class IfuToPreDecode(implicit p: Parameters) extends XSBundle { 81 val data = if(HasCExtension) Vec(PredictWidth + 1, UInt(16.W)) else Vec(PredictWidth, UInt(32.W)) 82 val frontendTrigger = new FrontendTdataDistributeIO 83 val csrTriggerEnable = Vec(4, Bool()) 84 val pc = Vec(PredictWidth, UInt(VAddrBits.W)) 85} 86 87 88class IfuToPredChecker(implicit p: Parameters) extends XSBundle { 89 val ftqOffset = Valid(UInt(log2Ceil(PredictWidth).W)) 90 val jumpOffset = Vec(PredictWidth, UInt(XLEN.W)) 91 val target = UInt(VAddrBits.W) 92 val instrRange = Vec(PredictWidth, Bool()) 93 val instrValid = Vec(PredictWidth, Bool()) 94 val pds = Vec(PredictWidth, new PreDecodeInfo) 95 val pc = Vec(PredictWidth, UInt(VAddrBits.W)) 96} 97 98class NewIFU(implicit p: Parameters) extends XSModule 99 with HasICacheParameters 100 with HasIFUConst 101 with HasPdConst 102 with HasCircularQueuePtrHelper 103 with HasPerfEvents 104{ 105 val io = IO(new NewIFUIO) 106 val (toFtq, fromFtq) = (io.ftqInter.toFtq, io.ftqInter.fromFtq) 107 val (toICache, fromICache) = (VecInit(io.icacheInter.map(_.req)), VecInit(io.icacheInter.map(_.resp))) 108 val (toUncache, fromUncache) = (io.uncacheInter.toUncache , io.uncacheInter.fromUncache) 109 110 def isCrossLineReq(start: UInt, end: UInt): Bool = start(blockOffBits) ^ end(blockOffBits) 111 112 def isLastInCacheline(addr: UInt): Bool = addr(blockOffBits - 1, 1) === 0.U 113 114 class TlbExept(implicit p: Parameters) extends XSBundle{ 115 val pageFault = Bool() 116 val accessFault = Bool() 117 val mmio = Bool() 118 } 119 120 val preDecoder = Module(new PreDecode) 121 val predChecker = Module(new PredChecker) 122 val frontendTrigger = Module(new FrontendTrigger) 123 val (preDecoderIn, preDecoderOut) = (preDecoder.io.in, preDecoder.io.out) 124 val (checkerIn, checkerOut) = (predChecker.io.in, predChecker.io.out) 125 126 io.iTLBInter.resp.ready := true.B 127 128 /** 129 ****************************************************************************** 130 * IFU Stage 0 131 * - send cacheline fetch request to ICacheMainPipe 132 ****************************************************************************** 133 */ 134 135 val f0_valid = fromFtq.req.valid 136 val f0_ftq_req = fromFtq.req.bits 137 val f0_doubleLine = fromFtq.req.bits.crossCacheline 138 val f0_vSetIdx = VecInit(get_idx((f0_ftq_req.startAddr)), get_idx(f0_ftq_req.nextlineStart)) 139 val f0_fire = fromFtq.req.fire() 140 141 val f0_flush, f1_flush, f2_flush, f3_flush = WireInit(false.B) 142 val from_bpu_f0_flush, from_bpu_f1_flush, from_bpu_f2_flush, from_bpu_f3_flush = WireInit(false.B) 143 144 from_bpu_f0_flush := fromFtq.flushFromBpu.shouldFlushByStage2(f0_ftq_req.ftqIdx) || 145 fromFtq.flushFromBpu.shouldFlushByStage3(f0_ftq_req.ftqIdx) 146 147 val wb_redirect , mmio_redirect, backend_redirect= WireInit(false.B) 148 val f3_wb_not_flush = WireInit(false.B) 149 150 backend_redirect := fromFtq.redirect.valid 151 f3_flush := backend_redirect || (wb_redirect && !f3_wb_not_flush) 152 f2_flush := backend_redirect || mmio_redirect || wb_redirect 153 f1_flush := f2_flush || from_bpu_f1_flush 154 f0_flush := f1_flush || from_bpu_f0_flush 155 156 val f1_ready, f2_ready, f3_ready = WireInit(false.B) 157 158 fromFtq.req.ready := toICache(0).ready && toICache(1).ready && f2_ready && GTimer() > 500.U 159 160 toICache(0).valid := fromFtq.req.valid //&& !f0_flush 161 toICache(0).bits.vaddr := fromFtq.req.bits.startAddr 162 toICache(1).valid := fromFtq.req.valid && f0_doubleLine //&& !f0_flush 163 toICache(1).bits.vaddr := fromFtq.req.bits.nextlineStart//fromFtq.req.bits.startAddr + (PredictWidth * 2).U //TODO: timing critical 164 165 /** <PERF> f0 fetch bubble */ 166 167 XSPerfAccumulate("fetch_bubble_ftq_not_valid", !f0_valid ) 168 XSPerfAccumulate("fetch_bubble_pipe_stall", f0_valid && toICache(0).ready && toICache(1).ready && !f1_ready ) 169 XSPerfAccumulate("fetch_bubble_sram_0_busy", f0_valid && !toICache(0).ready ) 170 XSPerfAccumulate("fetch_bubble_sram_1_busy", f0_valid && !toICache(1).ready ) 171 172 173 /** 174 ****************************************************************************** 175 * IFU Stage 1 176 * - calculate pc/half_pc/cut_ptr for every instruction 177 ****************************************************************************** 178 */ 179 180 val f1_valid = RegInit(false.B) 181 val f1_ftq_req = RegEnable(next = f0_ftq_req, enable=f0_fire) 182 // val f1_situation = RegEnable(next = f0_situation, enable=f0_fire) 183 val f1_doubleLine = RegEnable(next = f0_doubleLine, enable=f0_fire) 184 val f1_vSetIdx = RegEnable(next = f0_vSetIdx, enable=f0_fire) 185 val f1_fire = f1_valid && f1_ready 186 187 f1_ready := f2_ready || !f1_valid 188 189 from_bpu_f1_flush := fromFtq.flushFromBpu.shouldFlushByStage3(f1_ftq_req.ftqIdx) && f1_valid 190 // from_bpu_f1_flush := false.B 191 192 when(f1_flush) {f1_valid := false.B} 193 .elsewhen(f0_fire && !f0_flush) {f1_valid := true.B} 194 .elsewhen(f1_fire) {f1_valid := false.B} 195 196 val f1_pc = VecInit((0 until PredictWidth).map(i => f1_ftq_req.startAddr + (i * 2).U)) 197 val f1_half_snpc = VecInit((0 until PredictWidth).map(i => f1_ftq_req.startAddr + ((i+2) * 2).U)) 198 val f1_cut_ptr = if(HasCExtension) VecInit((0 until PredictWidth + 1).map(i => Cat(0.U(1.W), f1_ftq_req.startAddr(blockOffBits-1, 1)) + i.U )) 199 else VecInit((0 until PredictWidth).map(i => Cat(0.U(1.W), f1_ftq_req.startAddr(blockOffBits-1, 2)) + i.U )) 200 201 /** 202 ****************************************************************************** 203 * IFU Stage 2 204 * - icache response data (latched for pipeline stop) 205 * - generate exceprion bits for every instruciton (page fault/access fault/mmio) 206 * - generate predicted instruction range (1 means this instruciton is in this fetch packet) 207 * - cut data from cachlines to packet instruction code 208 * - instruction predecode and RVC expand 209 ****************************************************************************** 210 */ 211 212 val icacheRespAllValid = WireInit(false.B) 213 214 val f2_valid = RegInit(false.B) 215 val f2_ftq_req = RegEnable(next = f1_ftq_req, enable=f1_fire) 216 // val f2_situation = RegEnable(next = f1_situation, enable=f1_fire) 217 val f2_doubleLine = RegEnable(next = f1_doubleLine, enable=f1_fire) 218 val f2_vSetIdx = RegEnable(next = f1_vSetIdx, enable=f1_fire) 219 val f2_fire = f2_valid && f2_ready 220 221 f2_ready := f3_ready && icacheRespAllValid || !f2_valid 222 //TODO: addr compare may be timing critical 223 val f2_icache_all_resp_wire = fromICache(0).valid && (fromICache(0).bits.vaddr === f2_ftq_req.startAddr) && ((fromICache(1).valid && (fromICache(1).bits.vaddr === f2_ftq_req.nextlineStart)) || !f2_doubleLine) 224 val f2_icache_all_resp_reg = RegInit(false.B) 225 226 icacheRespAllValid := f2_icache_all_resp_reg || f2_icache_all_resp_wire 227 228 io.icacheStop := !f3_ready 229 230 when(f2_flush) {f2_icache_all_resp_reg := false.B} 231 .elsewhen(f2_valid && f2_icache_all_resp_wire && !f3_ready) {f2_icache_all_resp_reg := true.B} 232 .elsewhen(f2_fire && f2_icache_all_resp_reg) {f2_icache_all_resp_reg := false.B} 233 234 when(f2_flush) {f2_valid := false.B} 235 .elsewhen(f1_fire && !f1_flush) {f2_valid := true.B } 236 .elsewhen(f2_fire) {f2_valid := false.B} 237 238 val f2_cache_response_data = ResultHoldBypass(valid = f2_icache_all_resp_wire, data = VecInit(fromICache.map(_.bits.readData))) 239 240 val f2_except_pf = VecInit((0 until PortNumber).map(i => fromICache(i).bits.tlbExcp.pageFault)) 241 val f2_except_af = VecInit((0 until PortNumber).map(i => fromICache(i).bits.tlbExcp.accessFault)) 242 val f2_mmio = fromICache(0).bits.tlbExcp.mmio && !fromICache(0).bits.tlbExcp.accessFault && 243 !fromICache(0).bits.tlbExcp.pageFault 244 245 val f2_pc = RegEnable(next = f1_pc, enable = f1_fire) 246 val f2_half_snpc = RegEnable(next = f1_half_snpc, enable = f1_fire) 247 val f2_cut_ptr = RegEnable(next = f1_cut_ptr, enable = f1_fire) 248 249 val f2_resend_vaddr = RegEnable(next = f1_ftq_req.startAddr + 2.U, enable = f1_fire) 250 251 def isNextLine(pc: UInt, startAddr: UInt) = { 252 startAddr(blockOffBits) ^ pc(blockOffBits) 253 } 254 255 def isLastInLine(pc: UInt) = { 256 pc(blockOffBits - 1, 0) === "b111110".U 257 } 258 259 val f2_foldpc = VecInit(f2_pc.map(i => XORFold(i(VAddrBits-1,1), MemPredPCWidth))) 260 val f2_jump_range = Fill(PredictWidth, !f2_ftq_req.ftqOffset.valid) | Fill(PredictWidth, 1.U(1.W)) >> ~f2_ftq_req.ftqOffset.bits 261 val f2_ftr_range = Fill(PredictWidth, f2_ftq_req.oversize || f2_ftq_req.ftqOffset.valid) | Fill(PredictWidth, 1.U(1.W)) >> ~getBasicBlockIdx(f2_ftq_req.nextStartAddr, f2_ftq_req.startAddr) 262 val f2_instr_range = f2_jump_range & f2_ftr_range 263 val f2_pf_vec = VecInit((0 until PredictWidth).map(i => (!isNextLine(f2_pc(i), f2_ftq_req.startAddr) && f2_except_pf(0) || isNextLine(f2_pc(i), f2_ftq_req.startAddr) && f2_doubleLine && f2_except_pf(1)))) 264 val f2_af_vec = VecInit((0 until PredictWidth).map(i => (!isNextLine(f2_pc(i), f2_ftq_req.startAddr) && f2_except_af(0) || isNextLine(f2_pc(i), f2_ftq_req.startAddr) && f2_doubleLine && f2_except_af(1)))) 265 266 val f2_paddrs = VecInit((0 until PortNumber).map(i => fromICache(i).bits.paddr)) 267 val f2_perf_info = io.icachePerfInfo 268 269 def cut(cacheline: UInt, cutPtr: Vec[UInt]) : Vec[UInt] ={ 270 if(HasCExtension){ 271 val result = Wire(Vec(PredictWidth + 1, UInt(16.W))) 272 val dataVec = cacheline.asTypeOf(Vec(blockBytes * 2/ 2, UInt(16.W))) 273 (0 until PredictWidth + 1).foreach( i => 274 result(i) := dataVec(cutPtr(i)) 275 ) 276 result 277 } else { 278 val result = Wire(Vec(PredictWidth, UInt(32.W)) ) 279 val dataVec = cacheline.asTypeOf(Vec(blockBytes * 2/ 4, UInt(32.W))) 280 (0 until PredictWidth).foreach( i => 281 result(i) := dataVec(cutPtr(i)) 282 ) 283 result 284 } 285 } 286 287 val f2_datas = VecInit((0 until PortNumber).map(i => f2_cache_response_data(i))) 288 val f2_cut_data = cut( Cat(f2_datas.map(cacheline => cacheline.asUInt ).reverse).asUInt, f2_cut_ptr ) 289 290 /** predecode (include RVC expander) */ 291 preDecoderIn.data := f2_cut_data 292 preDecoderIn.frontendTrigger := io.frontendTrigger 293 preDecoderIn.csrTriggerEnable := io.csrTriggerEnable 294 preDecoderIn.pc := f2_pc 295 296 val f2_expd_instr = preDecoderOut.expInstr 297 val f2_pd = preDecoderOut.pd 298 val f2_jump_offset = preDecoderOut.jumpOffset 299 val f2_hasHalfValid = preDecoderOut.hasHalfValid 300 val f2_crossPageFault = VecInit((0 until PredictWidth).map(i => isLastInLine(f2_pc(i)) && !f2_except_pf(0) && f2_doubleLine && f2_except_pf(1) && !f2_pd(i).isRVC )) 301 302 val predecodeOutValid = WireInit(false.B) 303 304 305 /** 306 ****************************************************************************** 307 * IFU Stage 3 308 * - handle MMIO instruciton 309 * -send request to Uncache fetch Unit 310 * -every packet include 1 MMIO instruction 311 * -MMIO instructions will stop fetch pipeline until commiting from RoB 312 * -flush to snpc (send ifu_redirect to Ftq) 313 * - Ibuffer enqueue 314 * - check predict result in Frontend (jalFault/retFault/notCFIFault/invalidTakenFault/targetFault) 315 * - handle last half RVI instruction 316 ****************************************************************************** 317 */ 318 319 val f3_valid = RegInit(false.B) 320 val f3_ftq_req = RegEnable(next = f2_ftq_req, enable=f2_fire) 321 // val f3_situation = RegEnable(next = f2_situation, enable=f2_fire) 322 val f3_doubleLine = RegEnable(next = f2_doubleLine, enable=f2_fire) 323 val f3_fire = io.toIbuffer.fire() 324 325 f3_ready := io.toIbuffer.ready || !f3_valid 326 327 val f3_cut_data = RegEnable(next = f2_cut_data, enable=f2_fire) 328 329 val f3_except_pf = RegEnable(next = f2_except_pf, enable = f2_fire) 330 val f3_except_af = RegEnable(next = f2_except_af, enable = f2_fire) 331 val f3_mmio = RegEnable(next = f2_mmio , enable = f2_fire) 332 333 val f3_expd_instr = RegEnable(next = f2_expd_instr, enable = f2_fire) 334 val f3_pd = RegEnable(next = f2_pd, enable = f2_fire) 335 val f3_jump_offset = RegEnable(next = f2_jump_offset, enable = f2_fire) 336 val f3_af_vec = RegEnable(next = f2_af_vec, enable = f2_fire) 337 val f3_pf_vec = RegEnable(next = f2_pf_vec , enable = f2_fire) 338 val f3_pc = RegEnable(next = f2_pc, enable = f2_fire) 339 val f3_half_snpc = RegEnable(next = f2_half_snpc, enable = f2_fire) 340 val f3_instr_range = RegEnable(next = f2_instr_range, enable = f2_fire) 341 val f3_foldpc = RegEnable(next = f2_foldpc, enable = f2_fire) 342 val f3_crossPageFault = RegEnable(next = f2_crossPageFault, enable = f2_fire) 343 val f3_hasHalfValid = RegEnable(next = f2_hasHalfValid, enable = f2_fire) 344 val f3_except = VecInit((0 until 2).map{i => f3_except_pf(i) || f3_except_af(i)}) 345 val f3_has_except = f3_valid && (f3_except_af.reduce(_||_) || f3_except_pf.reduce(_||_)) 346 val f3_pAddrs = RegEnable(next = f2_paddrs, enable = f2_fire) 347 val f3_resend_vaddr = RegEnable(next = f2_resend_vaddr, enable = f2_fire) 348 349 350 val f3_oversize_target = f3_pc.last + 2.U 351 352 /*** MMIO State Machine***/ 353 val f3_mmio_data = Reg(Vec(2, UInt(16.W))) 354 val mmio_is_RVC = RegInit(false.B) 355 val mmio_resend_addr =RegInit(0.U(PAddrBits.W)) 356 val mmio_resend_af = RegInit(false.B) 357 358 val m_idle :: m_sendReq :: m_waitResp :: m_sendTLB :: m_tlbResp :: m_sendPMP :: m_resendReq :: m_waitResendResp :: m_waitCommit :: m_commited :: Nil = Enum(10) 359 val mmio_state = RegInit(m_idle) 360 361 val f3_req_is_mmio = f3_mmio && f3_valid 362 val mmio_commit = VecInit(io.rob_commits.map{commit => commit.valid && commit.bits.ftqIdx === f3_ftq_req.ftqIdx && commit.bits.ftqOffset === 0.U}).asUInt.orR 363 val f3_mmio_req_commit = f3_req_is_mmio && mmio_state === m_commited 364 365 val f3_mmio_to_commit = f3_req_is_mmio && mmio_state === m_waitCommit 366 val f3_mmio_to_commit_next = RegNext(f3_mmio_to_commit) 367 val f3_mmio_can_go = f3_mmio_to_commit && !f3_mmio_to_commit_next 368 369 val f3_ftq_flush_self = fromFtq.redirect.valid && RedirectLevel.flushItself(fromFtq.redirect.bits.level) 370 val f3_ftq_flush_by_older = fromFtq.redirect.valid && isBefore(fromFtq.redirect.bits.ftqIdx, f3_ftq_req.ftqIdx) 371 372 val f3_need_not_flush = f3_req_is_mmio && fromFtq.redirect.valid && !f3_ftq_flush_self && !f3_ftq_flush_by_older 373 374 when(f3_flush && !f3_need_not_flush) {f3_valid := false.B} 375 .elsewhen(f2_fire && !f2_flush ) {f3_valid := true.B } 376 .elsewhen(io.toIbuffer.fire() && !f3_req_is_mmio) {f3_valid := false.B} 377 .elsewhen{f3_req_is_mmio && f3_mmio_req_commit} {f3_valid := false.B} 378 379 val f3_mmio_use_seq_pc = RegInit(false.B) 380 381 val (redirect_ftqIdx, redirect_ftqOffset) = (fromFtq.redirect.bits.ftqIdx,fromFtq.redirect.bits.ftqOffset) 382 val redirect_mmio_req = fromFtq.redirect.valid && redirect_ftqIdx === f3_ftq_req.ftqIdx && redirect_ftqOffset === 0.U 383 384 when(RegNext(f2_fire && !f2_flush) && f3_req_is_mmio) { f3_mmio_use_seq_pc := true.B } 385 .elsewhen(redirect_mmio_req) { f3_mmio_use_seq_pc := false.B } 386 387 f3_ready := Mux(f3_req_is_mmio, io.toIbuffer.ready && f3_mmio_req_commit || !f3_valid , io.toIbuffer.ready || !f3_valid) 388 389 // when(fromUncache.fire()) {f3_mmio_data := fromUncache.bits.data} 390 391 392 switch(mmio_state){ 393 is(m_idle){ 394 when(f3_req_is_mmio){ 395 mmio_state := m_sendReq 396 } 397 } 398 399 is(m_sendReq){ 400 mmio_state := Mux(toUncache.fire(), m_waitResp, m_sendReq ) 401 } 402 403 is(m_waitResp){ 404 when(fromUncache.fire()){ 405 val isRVC = fromUncache.bits.data(1,0) =/= 3.U 406 val needResend = !isRVC && f3_pAddrs(0)(2,1) === 3.U 407 mmio_state := Mux(needResend, m_sendTLB , m_waitCommit) 408 409 mmio_is_RVC := isRVC 410 f3_mmio_data(0) := fromUncache.bits.data(15,0) 411 f3_mmio_data(1) := fromUncache.bits.data(31,16) 412 } 413 } 414 415 is(m_sendTLB){ 416 mmio_state := m_tlbResp 417 } 418 419 is(m_tlbResp){ 420 mmio_state := m_sendPMP 421 mmio_resend_addr := io.iTLBInter.resp.bits.paddr 422 } 423 424 is(m_sendPMP){ 425 val pmpExcpAF = io.pmp.resp.instr 426 mmio_state := Mux(pmpExcpAF, m_waitCommit , m_resendReq) 427 mmio_resend_af := pmpExcpAF 428 } 429 430 is(m_resendReq){ 431 mmio_state := Mux(toUncache.fire(), m_waitResendResp, m_resendReq ) 432 } 433 434 is(m_waitResendResp){ 435 when(fromUncache.fire()){ 436 mmio_state := m_waitCommit 437 f3_mmio_data(1) := fromUncache.bits.data(15,0) 438 } 439 } 440 441 is(m_waitCommit){ 442 when(mmio_commit){ 443 mmio_state := m_commited 444 } 445 } 446 447 //normal mmio instruction 448 is(m_commited){ 449 mmio_state := m_idle 450 mmio_is_RVC := false.B 451 mmio_resend_addr := 0.U 452 } 453 } 454 455 //exception or flush by older branch prediction 456 when(f3_ftq_flush_self || f3_ftq_flush_by_older) { 457 mmio_state := m_idle 458 mmio_is_RVC := false.B 459 mmio_resend_addr := 0.U 460 mmio_resend_af := false.B 461 f3_mmio_data.map(_ := 0.U) 462 } 463 464 toUncache.valid := ((mmio_state === m_sendReq) || (mmio_state === m_resendReq)) && f3_req_is_mmio 465 toUncache.bits.addr := Mux((mmio_state === m_resendReq), mmio_resend_addr, f3_pAddrs(0)) 466 fromUncache.ready := true.B 467 468 io.iTLBInter.req.valid := (mmio_state === m_sendTLB) && f3_req_is_mmio 469 io.iTLBInter.req.bits.size := 3.U 470 io.iTLBInter.req.bits.vaddr := f3_resend_vaddr 471 io.iTLBInter.req.bits.debug.pc := f3_resend_vaddr 472 473 io.iTLBInter.req.bits.cmd := TlbCmd.exec 474 io.iTLBInter.req.bits.robIdx := DontCare 475 io.iTLBInter.req.bits.debug.isFirstIssue := DontCare 476 477 io.pmp.req.valid := (mmio_state === m_sendPMP) && f3_req_is_mmio 478 io.pmp.req.bits.addr := mmio_resend_addr 479 io.pmp.req.bits.size := 3.U 480 io.pmp.req.bits.cmd := TlbCmd.exec 481 482 val f3_lastHalf = RegInit(0.U.asTypeOf(new LastHalfInfo)) 483 484 val f3_predecode_range = VecInit(preDecoderOut.pd.map(inst => inst.valid)).asUInt 485 val f3_mmio_range = VecInit((0 until PredictWidth).map(i => if(i ==0) true.B else false.B)) 486 val f3_instr_valid = Wire(Vec(PredictWidth, Bool())) 487 488 /*** prediction result check ***/ 489 checkerIn.ftqOffset := f3_ftq_req.ftqOffset 490 checkerIn.jumpOffset := f3_jump_offset 491 checkerIn.target := f3_ftq_req.nextStartAddr 492 checkerIn.instrRange := f3_instr_range.asTypeOf(Vec(PredictWidth, Bool())) 493 checkerIn.instrValid := f3_instr_valid.asTypeOf(Vec(PredictWidth, Bool())) 494 checkerIn.pds := f3_pd 495 checkerIn.pc := f3_pc 496 497 /*** handle half RVI in the last 2 Bytes ***/ 498 499 def hasLastHalf(idx: UInt) = { 500 !f3_pd(idx).isRVC && checkerOut.fixedRange(idx) && f3_instr_valid(idx) && !checkerOut.fixedTaken(idx) && !checkerOut.fixedMissPred(idx) && ! f3_req_is_mmio && !f3_ftq_req.oversize 501 } 502 503 val f3_last_validIdx = ~ParallelPriorityEncoder(checkerOut.fixedRange.reverse) 504 505 val f3_hasLastHalf = hasLastHalf((PredictWidth - 1).U) 506 val f3_false_lastHalf = hasLastHalf(f3_last_validIdx) 507 val f3_false_snpc = f3_half_snpc(f3_last_validIdx) 508 509 val f3_lastHalf_mask = VecInit((0 until PredictWidth).map( i => if(i ==0) false.B else true.B )).asUInt() 510 511 when (f3_flush) { 512 f3_lastHalf.valid := false.B 513 }.elsewhen (f3_fire) { 514 f3_lastHalf.valid := f3_hasLastHalf 515 f3_lastHalf.middlePC := f3_ftq_req.nextStartAddr 516 } 517 518 f3_instr_valid := Mux(f3_lastHalf.valid,f3_hasHalfValid ,VecInit(f3_pd.map(inst => inst.valid))) 519 520 /*** frontend Trigger ***/ 521 frontendTrigger.io.pds := f3_pd 522 frontendTrigger.io.pc := f3_pc 523 frontendTrigger.io.data := f3_cut_data 524 525 frontendTrigger.io.frontendTrigger := io.frontendTrigger 526 frontendTrigger.io.csrTriggerEnable := io.csrTriggerEnable 527 528 val f3_triggered = frontendTrigger.io.triggered 529 530 /*** send to Ibuffer ***/ 531 532 io.toIbuffer.valid := f3_valid && (!f3_req_is_mmio || f3_mmio_can_go) && !f3_flush 533 io.toIbuffer.bits.instrs := f3_expd_instr 534 io.toIbuffer.bits.valid := f3_instr_valid.asUInt 535 io.toIbuffer.bits.enqEnable := checkerOut.fixedRange.asUInt & f3_instr_valid.asUInt 536 io.toIbuffer.bits.pd := f3_pd 537 io.toIbuffer.bits.ftqPtr := f3_ftq_req.ftqIdx 538 io.toIbuffer.bits.pc := f3_pc 539 io.toIbuffer.bits.ftqOffset.zipWithIndex.map{case(a, i) => a.bits := i.U; a.valid := checkerOut.fixedTaken(i) && !f3_req_is_mmio} 540 io.toIbuffer.bits.foldpc := f3_foldpc 541 io.toIbuffer.bits.ipf := VecInit(f3_pf_vec.zip(f3_crossPageFault).map{case (pf, crossPF) => pf || crossPF}) 542 io.toIbuffer.bits.acf := f3_af_vec 543 io.toIbuffer.bits.crossPageIPFFix := f3_crossPageFault 544 io.toIbuffer.bits.triggered := f3_triggered 545 546 val lastHalfMask = VecInit((0 until PredictWidth).map(i => if(i ==0) false.B else true.B)) 547 when(f3_lastHalf.valid){ 548 io.toIbuffer.bits.enqEnable := checkerOut.fixedRange.asUInt & f3_instr_valid.asUInt & lastHalfMask.asUInt 549 io.toIbuffer.bits.valid := f3_lastHalf_mask & f3_instr_valid.asUInt 550 } 551 552 /** external predecode for MMIO instruction */ 553 when(f3_req_is_mmio){ 554 val inst = Cat(f3_mmio_data(1), f3_mmio_data(0)) 555 val currentIsRVC = isRVC(inst) 556 557 val brType::isCall::isRet::Nil = brInfo(inst) 558 val jalOffset = jal_offset(inst, currentIsRVC) 559 val brOffset = br_offset(inst, currentIsRVC) 560 561 io.toIbuffer.bits.instrs (0) := new RVCDecoder(inst, XLEN).decode.bits 562 563 io.toIbuffer.bits.pd(0).valid := true.B 564 io.toIbuffer.bits.pd(0).isRVC := currentIsRVC 565 io.toIbuffer.bits.pd(0).brType := brType 566 io.toIbuffer.bits.pd(0).isCall := isCall 567 io.toIbuffer.bits.pd(0).isRet := isRet 568 569 io.toIbuffer.bits.acf(0) := mmio_resend_af 570 571 io.toIbuffer.bits.enqEnable := f3_mmio_range.asUInt 572 } 573 574 575 //Write back to Ftq 576 val f3_cache_fetch = f3_valid && !(f2_fire && !f2_flush) 577 val finishFetchMaskReg = RegNext(f3_cache_fetch) 578 579 val mmioFlushWb = Wire(Valid(new PredecodeWritebackBundle)) 580 val f3_mmio_missOffset = Wire(ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))) 581 f3_mmio_missOffset.valid := f3_req_is_mmio 582 f3_mmio_missOffset.bits := 0.U 583 584 mmioFlushWb.valid := (f3_req_is_mmio && mmio_state === m_waitCommit && RegNext(fromUncache.fire()) && f3_mmio_use_seq_pc) 585 mmioFlushWb.bits.pc := f3_pc 586 mmioFlushWb.bits.pd := f3_pd 587 mmioFlushWb.bits.pd.zipWithIndex.map{case(instr,i) => instr.valid := f3_mmio_range(i)} 588 mmioFlushWb.bits.ftqIdx := f3_ftq_req.ftqIdx 589 mmioFlushWb.bits.ftqOffset := f3_ftq_req.ftqOffset.bits 590 mmioFlushWb.bits.misOffset := f3_mmio_missOffset 591 mmioFlushWb.bits.cfiOffset := DontCare 592 mmioFlushWb.bits.target := Mux(mmio_is_RVC, f3_ftq_req.startAddr + 2.U , f3_ftq_req.startAddr + 4.U) 593 mmioFlushWb.bits.jalTarget := DontCare 594 mmioFlushWb.bits.instrRange := f3_mmio_range 595 596 mmio_redirect := (f3_req_is_mmio && mmio_state === m_waitCommit && RegNext(fromUncache.fire()) && f3_mmio_use_seq_pc) 597 598 /** 599 ****************************************************************************** 600 * IFU Write Back Stage 601 * - write back predecode information to Ftq to update 602 * - redirect if found fault prediction 603 * - redirect if has false hit last half (last PC is not start + 32 Bytes, but in the midle of an notCFI RVI instruction) 604 ****************************************************************************** 605 */ 606 607 val wb_valid = RegNext(RegNext(f2_fire && !f2_flush) && !f3_req_is_mmio && !f3_flush) 608 val wb_ftq_req = RegNext(f3_ftq_req) 609 610 val wb_check_result = RegNext(checkerOut) 611 val wb_instr_range = RegNext(io.toIbuffer.bits.enqEnable) 612 val wb_pc = RegNext(f3_pc) 613 val wb_pd = RegNext(f3_pd) 614 val wb_instr_valid = RegNext(f3_instr_valid) 615 616 /* false hit lastHalf */ 617 val wb_lastIdx = RegNext(f3_last_validIdx) 618 val wb_false_lastHalf = RegNext(f3_false_lastHalf) && wb_lastIdx =/= (PredictWidth - 1).U 619 val wb_false_target = RegNext(f3_false_snpc) 620 621 val wb_half_flush = wb_false_lastHalf 622 val wb_half_target = wb_false_target 623 624 /* false oversize */ 625 val lastIsRVC = wb_instr_range.asTypeOf(Vec(PredictWidth,Bool())).last && wb_pd.last.isRVC 626 val lastIsRVI = wb_instr_range.asTypeOf(Vec(PredictWidth,Bool()))(PredictWidth - 2) && !wb_pd(PredictWidth - 2).isRVC 627 val lastTaken = wb_check_result.fixedTaken.last 628 val wb_false_oversize = wb_valid && wb_ftq_req.oversize && (lastIsRVC || lastIsRVI) && !lastTaken 629 val wb_oversize_target = RegNext(f3_oversize_target) 630 631 when(wb_valid){ 632 assert(!wb_false_oversize || !wb_half_flush, "False oversize and false half should be exclusive. ") 633 } 634 635 f3_wb_not_flush := wb_ftq_req.ftqIdx === f3_ftq_req.ftqIdx && f3_valid && wb_valid 636 637 val checkFlushWb = Wire(Valid(new PredecodeWritebackBundle)) 638 checkFlushWb.valid := wb_valid 639 checkFlushWb.bits.pc := wb_pc 640 checkFlushWb.bits.pd := wb_pd 641 checkFlushWb.bits.pd.zipWithIndex.map{case(instr,i) => instr.valid := wb_instr_valid(i)} 642 checkFlushWb.bits.ftqIdx := wb_ftq_req.ftqIdx 643 checkFlushWb.bits.ftqOffset := wb_ftq_req.ftqOffset.bits 644 checkFlushWb.bits.misOffset.valid := ParallelOR(wb_check_result.fixedMissPred) || wb_half_flush || wb_false_oversize 645 checkFlushWb.bits.misOffset.bits := Mux(wb_half_flush, (PredictWidth - 1).U, ParallelPriorityEncoder(wb_check_result.fixedMissPred)) 646 checkFlushWb.bits.cfiOffset.valid := ParallelOR(wb_check_result.fixedTaken) 647 checkFlushWb.bits.cfiOffset.bits := ParallelPriorityEncoder(wb_check_result.fixedTaken) 648 checkFlushWb.bits.target := Mux(wb_false_oversize, wb_oversize_target, 649 Mux(wb_half_flush, wb_half_target, wb_check_result.fixedTarget(ParallelPriorityEncoder(wb_check_result.fixedMissPred)))) 650 checkFlushWb.bits.jalTarget := wb_check_result.fixedTarget(ParallelPriorityEncoder(VecInit(wb_pd.zip(wb_instr_valid).map{case (pd, v) => v && pd.isJal }))) 651 checkFlushWb.bits.instrRange := wb_instr_range.asTypeOf(Vec(PredictWidth, Bool())) 652 653 toFtq.pdWb := Mux(f3_req_is_mmio, mmioFlushWb, checkFlushWb) 654 655 wb_redirect := checkFlushWb.bits.misOffset.valid && wb_valid 656 657 658 /** performance counter */ 659 val f3_perf_info = RegEnable(next = f2_perf_info, enable = f2_fire) 660 val f3_req_0 = io.toIbuffer.fire() 661 val f3_req_1 = io.toIbuffer.fire() && f3_doubleLine 662 val f3_hit_0 = io.toIbuffer.fire() && f3_perf_info.bank_hit(0) 663 val f3_hit_1 = io.toIbuffer.fire() && f3_doubleLine & f3_perf_info.bank_hit(1) 664 val f3_hit = f3_perf_info.hit 665 val perfEvents = Seq( 666 ("frontendFlush ", wb_redirect ), 667 ("ifu_req ", io.toIbuffer.fire() ), 668 ("ifu_miss ", io.toIbuffer.fire() && !f3_perf_info.hit ), 669 ("ifu_req_cacheline_0 ", f3_req_0 ), 670 ("ifu_req_cacheline_1 ", f3_req_1 ), 671 ("ifu_req_cacheline_0_hit ", f3_hit_1 ), 672 ("ifu_req_cacheline_1_hit ", f3_hit_1 ), 673 ("only_0_hit ", f3_perf_info.only_0_hit && io.toIbuffer.fire() ), 674 ("only_0_miss ", f3_perf_info.only_0_miss && io.toIbuffer.fire() ), 675 ("hit_0_hit_1 ", f3_perf_info.hit_0_hit_1 && io.toIbuffer.fire() ), 676 ("hit_0_miss_1 ", f3_perf_info.hit_0_miss_1 && io.toIbuffer.fire() ), 677 ("miss_0_hit_1 ", f3_perf_info.miss_0_hit_1 && io.toIbuffer.fire() ), 678 ("miss_0_miss_1 ", f3_perf_info.miss_0_miss_1 && io.toIbuffer.fire() ), 679 // ("cross_line_block ", io.toIbuffer.fire() && f3_situation(0) ), 680 // ("fall_through_is_cacheline_end", io.toIbuffer.fire() && f3_situation(1) ), 681 ) 682 generatePerfEvent() 683 684 XSPerfAccumulate("ifu_req", io.toIbuffer.fire() ) 685 XSPerfAccumulate("ifu_miss", io.toIbuffer.fire() && !f3_hit ) 686 XSPerfAccumulate("ifu_req_cacheline_0", f3_req_0 ) 687 XSPerfAccumulate("ifu_req_cacheline_1", f3_req_1 ) 688 XSPerfAccumulate("ifu_req_cacheline_0_hit", f3_hit_0 ) 689 XSPerfAccumulate("ifu_req_cacheline_1_hit", f3_hit_1 ) 690 XSPerfAccumulate("frontendFlush", wb_redirect ) 691 XSPerfAccumulate("only_0_hit", f3_perf_info.only_0_hit && io.toIbuffer.fire() ) 692 XSPerfAccumulate("only_0_miss", f3_perf_info.only_0_miss && io.toIbuffer.fire() ) 693 XSPerfAccumulate("hit_0_hit_1", f3_perf_info.hit_0_hit_1 && io.toIbuffer.fire() ) 694 XSPerfAccumulate("hit_0_miss_1", f3_perf_info.hit_0_miss_1 && io.toIbuffer.fire() ) 695 XSPerfAccumulate("miss_0_hit_1", f3_perf_info.miss_0_hit_1 && io.toIbuffer.fire() ) 696 XSPerfAccumulate("miss_0_miss_1", f3_perf_info.miss_0_miss_1 && io.toIbuffer.fire() ) 697 XSPerfAccumulate("hit_0_except_1", f3_perf_info.hit_0_except_1 && io.toIbuffer.fire() ) 698 XSPerfAccumulate("miss_0_except_1", f3_perf_info.miss_0_except_1 && io.toIbuffer.fire() ) 699 XSPerfAccumulate("except_0", f3_perf_info.except_0 && io.toIbuffer.fire() ) 700} 701