1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import xiangshan._ 24import xiangshan.backend.CtrlToFtqIO 25 26class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr]( 27 p => p(XSCoreParamsKey).FtqSize 28){ 29 override def cloneType = (new FtqPtr).asInstanceOf[this.type] 30} 31 32object FtqPtr { 33 def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = { 34 val ptr = Wire(new FtqPtr) 35 ptr.flag := f 36 ptr.value := v 37 ptr 38 } 39 def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = { 40 apply(!ptr.flag, ptr.value) 41 } 42} 43 44class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule { 45 46 val io = IO(new Bundle() { 47 val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W))) 48 val ren = Input(Vec(numRead, Bool())) 49 val rdata = Output(Vec(numRead, gen)) 50 val waddr = Input(UInt(log2Up(FtqSize).W)) 51 val wen = Input(Bool()) 52 val wdata = Input(gen) 53 }) 54 55 for(i <- 0 until numRead){ 56 val sram = Module(new SRAMTemplate(gen, FtqSize)) 57 sram.io.r.req.valid := io.ren(i) 58 sram.io.r.req.bits.setIdx := io.raddr(i) 59 io.rdata(i) := sram.io.r.resp.data(0) 60 sram.io.w.req.valid := io.wen 61 sram.io.w.req.bits.setIdx := io.waddr 62 sram.io.w.req.bits.data := VecInit(io.wdata) 63 } 64 65} 66 67class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils { 68 // TODO: move pftAddr, oversize, carry to another mem 69 val startAddr = UInt(VAddrBits.W) 70 val nextLineAddr = UInt(VAddrBits.W) 71 val isNextMask = Vec(PredictWidth, Bool()) 72 val oversize = Bool() 73 val fallThruError = Bool() 74 // val carry = Bool() 75 def getPc(offset: UInt) = { 76 def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1) 77 def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits) 78 Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)), 79 getOffset(startAddr)+offset, 0.U(instOffsetBits.W)) 80 } 81 def fromBranchPrediction(resp: BranchPredictionBundle) = { 82 def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1) 83 this.startAddr := resp.pc 84 this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U 85 this.isNextMask := VecInit((0 until PredictWidth).map(i => 86 (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool() 87 )) 88 this.oversize := resp.oversize 89 this.fallThruError := resp.fallThruError 90 this 91 } 92 override def toPrintable: Printable = { 93 p"startAddr:${Hexadecimal(startAddr)}" 94 } 95} 96 97class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle { 98 val brMask = Vec(PredictWidth, Bool()) 99 val jmpInfo = ValidUndirectioned(Vec(3, Bool())) 100 val jmpOffset = UInt(log2Ceil(PredictWidth).W) 101 val jalTarget = UInt(VAddrBits.W) 102 val rvcMask = Vec(PredictWidth, Bool()) 103 def hasJal = jmpInfo.valid && !jmpInfo.bits(0) 104 def hasJalr = jmpInfo.valid && jmpInfo.bits(0) 105 def hasCall = jmpInfo.valid && jmpInfo.bits(1) 106 def hasRet = jmpInfo.valid && jmpInfo.bits(2) 107 108 def fromPdWb(pdWb: PredecodeWritebackBundle) = { 109 val pds = pdWb.pd 110 this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid)) 111 this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR 112 this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid), 113 pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet))) 114 this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)) 115 this.rvcMask := VecInit(pds.map(pd => pd.isRVC)) 116 this.jalTarget := pdWb.jalTarget 117 } 118 119 def toPd(offset: UInt) = { 120 require(offset.getWidth == log2Ceil(PredictWidth)) 121 val pd = Wire(new PreDecodeInfo) 122 pd.valid := true.B 123 pd.isRVC := rvcMask(offset) 124 val isBr = brMask(offset) 125 val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0) 126 pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr) 127 pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1) 128 pd.isRet := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2) 129 pd 130 } 131} 132 133 134 135class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 136 val rasSp = UInt(log2Ceil(RasSize).W) 137 val rasEntry = new RASEntry 138 // val specCnt = Vec(numBr, UInt(10.W)) 139 // val ghist = new ShiftingGlobalHistory 140 val folded_hist = new AllFoldedHistories(foldedGHistInfos) 141 val histPtr = new CGHPtr 142 143 def fromBranchPrediction(resp: BranchPredictionBundle) = { 144 assert(!resp.is_minimal) 145 this.rasSp := resp.rasSp 146 this.rasEntry := resp.rasTop 147 this.folded_hist := resp.folded_hist 148 this.histPtr := resp.histPtr 149 this 150 } 151} 152 153class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 154 val meta = UInt(MaxMetaLength.W) 155} 156 157class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle { 158 val target = UInt(VAddrBits.W) 159 val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)) 160} 161 162// class FtqEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 163// val startAddr = UInt(VAddrBits.W) 164// val fallThruAddr = UInt(VAddrBits.W) 165// val isNextMask = Vec(PredictWidth, Bool()) 166 167// val meta = UInt(MaxMetaLength.W) 168 169// val rasSp = UInt(log2Ceil(RasSize).W) 170// val rasEntry = new RASEntry 171// val hist = new ShiftingGlobalHistory 172// val specCnt = Vec(numBr, UInt(10.W)) 173 174// val valids = Vec(PredictWidth, Bool()) 175// val brMask = Vec(PredictWidth, Bool()) 176// // isJalr, isCall, isRet 177// val jmpInfo = ValidUndirectioned(Vec(3, Bool())) 178// val jmpOffset = UInt(log2Ceil(PredictWidth).W) 179 180// val mispredVec = Vec(PredictWidth, Bool()) 181// val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)) 182// val target = UInt(VAddrBits.W) 183// } 184 185class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle { 186 val ptr = Output(new FtqPtr) 187 val offset = Output(UInt(log2Ceil(PredictWidth).W)) 188 val data = Input(gen) 189 def apply(ptr: FtqPtr, offset: UInt) = { 190 this.ptr := ptr 191 this.offset := offset 192 this.data 193 } 194 override def cloneType = (new FtqRead(gen)).asInstanceOf[this.type] 195} 196 197 198class FtqToBpuIO(implicit p: Parameters) extends XSBundle { 199 val redirect = Valid(new BranchPredictionRedirect) 200 val update = Valid(new BranchPredictionUpdate) 201 val enq_ptr = Output(new FtqPtr) 202} 203 204class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper { 205 val req = Decoupled(new FetchRequestBundle) 206 val redirect = Valid(new Redirect) 207 val flushFromBpu = new Bundle { 208 // when ifu pipeline is not stalled, 209 // a packet from bpu s3 can reach f1 at most 210 val s2 = Valid(new FtqPtr) 211 // val s3 = Valid(new FtqPtr) 212 def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = { 213 src.valid && !isAfter(src.bits, idx_to_flush) 214 } 215 def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx) 216 // def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx) 217 } 218} 219 220trait HasBackendRedirectInfo extends HasXSParameter { 221 def numRedirect = exuParameters.JmpCnt + exuParameters.AluCnt + 1 222 def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself() 223} 224 225class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo { 226 val pc_reads = Vec(1 + numRedirect + 1 + 1, Flipped(new FtqRead(UInt(VAddrBits.W)))) 227 val target_read = Flipped(new FtqRead(UInt(VAddrBits.W))) 228 def getJumpPcRead = pc_reads.head 229 def getRedirectPcRead = VecInit(pc_reads.tail.dropRight(2)) 230 def getMemPredPcRead = pc_reads.init.last 231 def getRobFlushPcRead = pc_reads.last 232} 233 234 235class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter { 236 val io = IO(new Bundle { 237 val start_addr = Input(UInt(VAddrBits.W)) 238 val old_entry = Input(new FTBEntry) 239 val pd = Input(new Ftq_pd_Entry) 240 val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W))) 241 val target = Input(UInt(VAddrBits.W)) 242 val hit = Input(Bool()) 243 val mispredict_vec = Input(Vec(PredictWidth, Bool())) 244 245 val new_entry = Output(new FTBEntry) 246 val new_br_insert_pos = Output(Vec(numBr, Bool())) 247 val taken_mask = Output(Vec(numBr, Bool())) 248 val mispred_mask = Output(Vec(numBr+1, Bool())) 249 250 // for perf counters 251 val is_init_entry = Output(Bool()) 252 val is_old_entry = Output(Bool()) 253 val is_new_br = Output(Bool()) 254 val is_jalr_target_modified = Output(Bool()) 255 val is_always_taken_modified = Output(Bool()) 256 val is_br_full = Output(Bool()) 257 }) 258 259 // no mispredictions detected at predecode 260 val hit = io.hit 261 val pd = io.pd 262 263 val init_entry = WireInit(0.U.asTypeOf(new FTBEntry)) 264 265 266 val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid 267 val entry_has_jmp = pd.jmpInfo.valid 268 val new_jmp_is_jal = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid 269 val new_jmp_is_jalr = entry_has_jmp && pd.jmpInfo.bits(0) && io.cfiIndex.valid 270 val new_jmp_is_call = entry_has_jmp && pd.jmpInfo.bits(1) && io.cfiIndex.valid 271 val new_jmp_is_ret = entry_has_jmp && pd.jmpInfo.bits(2) && io.cfiIndex.valid 272 val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last 273 val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last 274 275 val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal 276 val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr 277 278 def carryPos = log2Ceil(PredictWidth)+instOffsetBits+1 279 def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits) 280 // if not hit, establish a new entry 281 init_entry.valid := true.B 282 // tag is left for ftb to assign 283 284 // case br 285 val init_br_slot = init_entry.getSlotForBr(0) 286 when (cfi_is_br) { 287 init_br_slot.valid := true.B 288 init_br_slot.offset := io.cfiIndex.bits 289 init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1) 290 init_entry.always_taken(0) := true.B // set to always taken on init 291 } 292 293 // case jmp 294 when (entry_has_jmp) { 295 init_entry.tailSlot.offset := pd.jmpOffset 296 init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr 297 init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false) 298 } 299 300 val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U) 301 init_entry.pftAddr := Mux(entry_has_jmp, jmpPft, getLower(io.start_addr) + ((FetchWidth*4)>>instOffsetBits).U + Mux(last_br_rvi, 1.U, 0.U)) 302 init_entry.carry := Mux(entry_has_jmp, jmpPft(carryPos-instOffsetBits), io.start_addr(carryPos-1) || (io.start_addr(carryPos-2, instOffsetBits).andR && last_br_rvi)) 303 init_entry.isJalr := new_jmp_is_jalr 304 init_entry.isCall := new_jmp_is_call 305 init_entry.isRet := new_jmp_is_ret 306 init_entry.last_is_rvc := Mux(entry_has_jmp, pd.rvcMask(pd.jmpOffset), pd.rvcMask.last) 307 308 init_entry.oversize := last_br_rvi || last_jmp_rvi 309 310 // if hit, check whether a new cfi(only br is possible) is detected 311 val oe = io.old_entry 312 val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits) 313 val br_recorded = br_recorded_vec.asUInt.orR 314 val is_new_br = cfi_is_br && !br_recorded 315 val new_br_offset = io.cfiIndex.bits 316 // vec(i) means new br will be inserted BEFORE old br(i) 317 val allBrSlotsVec = oe.allSlotsForBr 318 val new_br_insert_onehot = VecInit((0 until numBr).map{ 319 i => i match { 320 case 0 => 321 !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset 322 case idx => 323 allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset && 324 (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset) 325 } 326 }) 327 328 val old_entry_modified = WireInit(io.old_entry) 329 for (i <- 0 until numBr) { 330 val slot = old_entry_modified.allSlotsForBr(i) 331 when (new_br_insert_onehot(i)) { 332 slot.valid := true.B 333 slot.offset := new_br_offset 334 slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1) 335 old_entry_modified.always_taken(i) := true.B 336 }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) { 337 old_entry_modified.always_taken(i) := false.B 338 // all other fields remain unchanged 339 }.otherwise { 340 // case i == 0, remain unchanged 341 if (i != 0) { 342 val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid 343 when (!noNeedToMoveFromFormerSlot) { 344 slot.fromAnotherSlot(oe.allSlotsForBr(i-1)) 345 old_entry_modified.always_taken(i) := oe.always_taken(i) 346 } 347 } 348 } 349 } 350 351 // two circumstances: 352 // 1. oe: | br | j |, new br should be in front of j, thus addr of j should be new pft 353 // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either 354 // the previous last br or the new br 355 val may_have_to_replace = oe.noEmptySlotForNewBr 356 val pft_need_to_change = is_new_br && may_have_to_replace 357 // it should either be the given last br or the new br 358 when (pft_need_to_change) { 359 val new_pft_offset = 360 Mux(!new_br_insert_onehot.asUInt.orR, 361 new_br_offset, oe.allSlotsForBr.last.offset) 362 363 // set jmp to invalid 364 old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset 365 old_entry_modified.last_is_rvc := pd.rvcMask(new_pft_offset - 1.U) // TODO: fix this 366 old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool 367 old_entry_modified.oversize := false.B 368 old_entry_modified.isCall := false.B 369 old_entry_modified.isRet := false.B 370 old_entry_modified.isJalr := false.B 371 } 372 373 val old_entry_jmp_target_modified = WireInit(oe) 374 val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits 375 val old_tail_is_jmp = !oe.tailSlot.sharing 376 val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target 377 when (jalr_target_modified) { 378 old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target) 379 old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool())) 380 } 381 382 val old_entry_always_taken = WireInit(oe) 383 val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not 384 for (i <- 0 until numBr) { 385 old_entry_always_taken.always_taken(i) := 386 oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i) 387 always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i) 388 } 389 val always_taken_modified = always_taken_modified_vec.reduce(_||_) 390 391 392 393 val derived_from_old_entry = 394 Mux(is_new_br, old_entry_modified, 395 Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken)) 396 397 398 io.new_entry := Mux(!hit, init_entry, derived_from_old_entry) 399 400 io.new_br_insert_pos := new_br_insert_onehot 401 io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{ 402 case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v 403 }) 404 for (i <- 0 until numBr) { 405 io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i)) 406 } 407 io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset) 408 409 // for perf counters 410 io.is_init_entry := !hit 411 io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified 412 io.is_new_br := hit && is_new_br 413 io.is_jalr_target_modified := hit && jalr_target_modified 414 io.is_always_taken_modified := hit && always_taken_modified 415 io.is_br_full := hit && is_new_br && may_have_to_replace 416} 417 418class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper 419 with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents { 420 val io = IO(new Bundle { 421 val fromBpu = Flipped(new BpuToFtqIO) 422 val fromIfu = Flipped(new IfuToFtqIO) 423 val fromBackend = Flipped(new CtrlToFtqIO) 424 425 val toBpu = new FtqToBpuIO 426 val toIfu = new FtqToIfuIO 427 val toBackend = new FtqToCtrlIO 428 429 val toPrefetch = new FtqPrefechBundle 430 431 val bpuInfo = new Bundle { 432 val bpRight = Output(UInt(XLEN.W)) 433 val bpWrong = Output(UInt(XLEN.W)) 434 } 435 }) 436 io.bpuInfo := DontCare 437 438 val backendRedirect = io.fromBackend.redirect 439 val backendRedirectReg = RegNext(io.fromBackend.redirect) 440 441 val stage2Flush = backendRedirect.valid 442 val backendFlush = stage2Flush || RegNext(stage2Flush) 443 val ifuFlush = Wire(Bool()) 444 445 val flush = stage2Flush || RegNext(stage2Flush) 446 447 val allowBpuIn, allowToIfu = WireInit(false.B) 448 val flushToIfu = !allowToIfu 449 allowBpuIn := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid 450 allowToIfu := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid 451 452 val bpuPtr, ifuPtr, ifuWbPtr, commPtr, prefetchPtr = RegInit(FtqPtr(false.B, 0.U)) 453 val validEntries = distanceBetween(bpuPtr, commPtr) 454 455 // ********************************************************************** 456 // **************************** enq from bpu **************************** 457 // ********************************************************************** 458 val new_entry_ready = validEntries < FtqSize.U 459 io.fromBpu.resp.ready := new_entry_ready 460 461 val bpu_s2_resp = io.fromBpu.resp.bits.s2 462 // val bpu_s3_resp = io.fromBpu.resp.bits.s3 463 val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect 464 // val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect 465 466 io.toBpu.enq_ptr := bpuPtr 467 val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1 468 val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect/* || bpu_s3_redirect */) && allowBpuIn 469 470 val bpu_in_resp = io.fromBpu.resp.bits.selectedResp 471 val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx 472 val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx) 473 val bpu_in_resp_idx = bpu_in_resp_ptr.value 474 475 // read ports: jumpPc + redirects + loadPred + robFlush + ifuReq1 + ifuReq2 + commitUpdate 476 val ftq_pc_mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize, 1+numRedirect+2+1+1+1, 1)) 477 // resp from uBTB 478 ftq_pc_mem.io.wen(0) := bpu_in_fire 479 ftq_pc_mem.io.waddr(0) := bpu_in_resp_idx 480 ftq_pc_mem.io.wdata(0).fromBranchPrediction(bpu_in_resp) 481 482 // ifuRedirect + backendRedirect + commit 483 val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1)) 484 // these info is intended to enq at the last stage of bpu 485 ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid 486 ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value 487 ftq_redirect_sram.io.wdata.fromBranchPrediction(io.fromBpu.resp.bits.lastStage) 488 489 val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1)) 490 // these info is intended to enq at the last stage of bpu 491 ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid 492 ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value 493 ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.meta 494 // ifuRedirect + backendRedirect + commit 495 val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1)) 496 ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid 497 ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value 498 ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.lastStage.ftb_entry 499 500 501 // multi-write 502 val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough 503 val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)))) 504 val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool()))) 505 val pred_stage = Reg(Vec(FtqSize, UInt(2.W))) 506 507 val c_invalid :: c_valid :: c_commited :: Nil = Enum(3) 508 val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) { 509 VecInit(Seq.fill(PredictWidth)(c_invalid)) 510 })) 511 512 val f_to_send :: f_sent :: Nil = Enum(2) 513 val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent))) 514 515 val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3) 516 val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit))) 517 518 519 when (bpu_in_fire) { 520 entry_fetch_status(bpu_in_resp_idx) := f_to_send 521 commitStateQueue(bpu_in_resp_idx) := VecInit(Seq.fill(PredictWidth)(c_invalid)) 522 cfiIndex_vec(bpu_in_resp_idx) := bpu_in_resp.cfiIndex 523 mispredict_vec(bpu_in_resp_idx) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B))) 524 update_target(bpu_in_resp_idx) := bpu_in_resp.getTarget 525 pred_stage(bpu_in_resp_idx) := bpu_in_stage 526 } 527 528 bpuPtr := bpuPtr + enq_fire 529 ifuPtr := ifuPtr + io.toIfu.req.fire 530 prefetchPtr := prefetchPtr + io.toPrefetch.req.fire() 531 532 // only use ftb result to assign hit status 533 when (bpu_s2_resp.valid) { 534 entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit) 535 } 536 537 538 io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect 539 io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx 540 when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) { 541 bpuPtr := bpu_s2_resp.ftq_idx + 1.U 542 // only when ifuPtr runs ahead of bpu s2 resp should we recover it 543 when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) { 544 ifuPtr := bpu_s2_resp.ftq_idx 545 } 546 547 when (!isBefore(prefetchPtr, bpu_s2_resp.ftq_idx)) { 548 prefetchPtr := bpu_s2_resp.ftq_idx 549 } 550 } 551 552 // io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect 553 // io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx 554 // when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) { 555 // bpuPtr := bpu_s3_resp.ftq_idx + 1.U 556 // // only when ifuPtr runs ahead of bpu s2 resp should we recover it 557 // when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) { 558 // ifuPtr := bpu_s3_resp.ftq_idx 559 // } 560 // XSError(true.B, "\ns3_redirect mechanism not implemented!\n") 561 // } 562 563 XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n") 564 XSError(isBefore(bpuPtr, prefetchPtr) && !isFull(bpuPtr, prefetchPtr), "\nprefetchPtr is before bpuPtr!\n") 565 566 // **************************************************************** 567 // **************************** to ifu **************************** 568 // **************************************************************** 569 val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata(0), enable=bpu_in_fire) 570 val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr) 571 val last_cycle_bpu_in = RegNext(bpu_in_fire) 572 val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire) 573 574 // read pc and target 575 ftq_pc_mem.io.raddr.init.init.last := ifuPtr.value 576 ftq_pc_mem.io.raddr.init.last := (ifuPtr+1.U).value 577 578 io.toIfu.req.valid := allowToIfu && entry_fetch_status(ifuPtr.value) === f_to_send && ifuPtr =/= bpuPtr 579 io.toIfu.req.bits.ftqIdx := ifuPtr 580 io.toIfu.req.bits.nextStartAddr := update_target(ifuPtr.value) 581 io.toIfu.req.bits.ftqOffset := cfiIndex_vec(ifuPtr.value) 582 583 val toIfuPcBundle = Wire(new Ftq_RF_Components) 584 585 when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) { 586 toIfuPcBundle := bpu_in_bypass_buf 587 }.elsewhen (last_cycle_to_ifu_fire) { 588 toIfuPcBundle := ftq_pc_mem.io.rdata.init.last 589 }.otherwise { 590 toIfuPcBundle := ftq_pc_mem.io.rdata.init.init.last 591 } 592 593 io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle) 594 595 // when fall through is smaller in value than start address, there must be a false hit 596 when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) { 597 when (io.toIfu.req.fire && 598 !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr)/* && 599 !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr) */ 600 ) { 601 entry_hit_status(ifuPtr.value) := h_false_hit 602 // XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr) 603 } 604 XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr) 605 } 606 607 val ifu_req_should_be_flushed = 608 io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx)/* || 609 io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx) */ 610 611 when (io.toIfu.req.fire && !ifu_req_should_be_flushed) { 612 entry_fetch_status(ifuPtr.value) := f_sent 613 } 614 615 io.toPrefetch.req.valid := allowToIfu && prefetchPtr =/= bpuPtr && entry_fetch_status(prefetchPtr.value) === f_to_send 616 io.toPrefetch.req.bits.target := update_target(prefetchPtr.value) 617 618 // ********************************************************************* 619 // **************************** wb from ifu **************************** 620 // ********************************************************************* 621 val pdWb = io.fromIfu.pdWb 622 val pds = pdWb.bits.pd 623 val ifu_wb_valid = pdWb.valid 624 val ifu_wb_idx = pdWb.bits.ftqIdx.value 625 // read ports: commit update 626 val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1)) 627 ftq_pd_mem.io.wen(0) := ifu_wb_valid 628 ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value 629 ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits) 630 631 val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid 632 val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid 633 val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B) 634 val pd_reg = RegEnable(pds, enable = pdWb.valid) 635 val start_pc_reg = RegEnable(pdWb.bits.pc(0), enable = pdWb.valid) 636 val wb_idx_reg = RegEnable(ifu_wb_idx, enable = pdWb.valid) 637 638 when (ifu_wb_valid) { 639 val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{ 640 case (v, inRange) => v && inRange 641 }) 642 (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{ 643 case (qe, v) => when (v) { qe := c_valid } 644 } 645 } 646 647 ifuWbPtr := ifuWbPtr + ifu_wb_valid 648 649 ftb_entry_mem.io.raddr.head := ifu_wb_idx 650 val has_false_hit = WireInit(false.B) 651 when (RegNext(hit_pd_valid)) { 652 // check for false hit 653 val pred_ftb_entry = ftb_entry_mem.io.rdata.head 654 val brSlots = pred_ftb_entry.brSlots 655 val tailSlot = pred_ftb_entry.tailSlot 656 // we check cfis that bpu predicted 657 658 // bpu predicted branches but denied by predecode 659 val br_false_hit = 660 brSlots.map{ 661 s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr) 662 }.reduce(_||_) || 663 (tailSlot.valid && pred_ftb_entry.tailSlot.sharing && 664 !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr)) 665 666 val jmpOffset = tailSlot.offset 667 val jmp_pd = pd_reg(jmpOffset) 668 val jal_false_hit = pred_ftb_entry.jmpValid && 669 ((pred_ftb_entry.isJal && !(jmp_pd.valid && jmp_pd.isJal)) || 670 (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) || 671 (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) || 672 (pred_ftb_entry.isRet && !(jmp_pd.valid && jmp_pd.isRet)) 673 ) 674 675 has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg 676 XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0)) 677 678 // assert(!has_false_hit) 679 } 680 681 when (has_false_hit) { 682 entry_hit_status(wb_idx_reg) := h_false_hit 683 } 684 685 686 // ********************************************************************** 687 // **************************** backend read **************************** 688 // ********************************************************************** 689 690 // pc reads 691 for ((req, i) <- io.toBackend.pc_reads.zipWithIndex) { 692 ftq_pc_mem.io.raddr(i) := req.ptr.value 693 req.data := ftq_pc_mem.io.rdata(i).getPc(RegNext(req.offset)) 694 } 695 // target read 696 io.toBackend.target_read.data := RegNext(update_target(io.toBackend.target_read.ptr.value)) 697 698 // ******************************************************************************* 699 // **************************** redirect from backend **************************** 700 // ******************************************************************************* 701 702 // redirect read cfiInfo, couples to redirectGen s2 703 ftq_redirect_sram.io.ren.init.last := io.fromBackend.redirect.valid 704 ftq_redirect_sram.io.raddr.init.last := io.fromBackend.redirect.bits.ftqIdx.value 705 706 ftb_entry_mem.io.raddr.init.last := io.fromBackend.redirect.bits.ftqIdx.value 707 708 val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last 709 val fromBackendRedirect = WireInit(backendRedirectReg) 710 val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate 711 backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo) 712 713 val r_ftb_entry = ftb_entry_mem.io.rdata.init.last 714 val r_ftqOffset = fromBackendRedirect.bits.ftqOffset 715 716 when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) { 717 backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +& 718 (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) && 719 !r_ftb_entry.newBrCanNotInsert(r_ftqOffset)) 720 721 backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) || 722 !r_ftb_entry.newBrCanNotInsert(r_ftqOffset)) 723 }.otherwise { 724 backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt 725 backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt 726 } 727 728 729 // *************************************************************************** 730 // **************************** redirect from ifu **************************** 731 // *************************************************************************** 732 val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect))) 733 fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush 734 fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx 735 fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits 736 fromIfuRedirect.bits.level := RedirectLevel.flushAfter 737 738 val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate 739 ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits) 740 ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits) 741 ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid 742 ifuRedirectCfiUpdate.target := pdWb.bits.target 743 ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid 744 ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid 745 746 val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect))) 747 val ifuRedirectToBpu = WireInit(ifuRedirectReg) 748 ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid 749 750 ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid 751 ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value 752 753 ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value 754 755 val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate 756 toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head) 757 when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) { 758 toBpuCfi.target := toBpuCfi.rasEntry.retAddr 759 } 760 761 // ********************************************************************* 762 // **************************** wb from exu **************************** 763 // ********************************************************************* 764 765 def extractRedirectInfo(wb: Valid[Redirect]) = { 766 val ftqIdx = wb.bits.ftqIdx.value 767 val ftqOffset = wb.bits.ftqOffset 768 val taken = wb.bits.cfiUpdate.taken 769 val mispred = wb.bits.cfiUpdate.isMisPred 770 (wb.valid, ftqIdx, ftqOffset, taken, mispred) 771 } 772 773 // fix mispredict entry 774 val lastIsMispredict = RegNext( 775 backendRedirect.valid && backendRedirect.bits.level === RedirectLevel.flushAfter, init = false.B 776 ) 777 778 def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = { 779 val (r_valid, r_idx, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect) 780 val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits 781 val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits 782 when (cfiIndex_bits_wen || cfiIndex_valid_wen) { 783 cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken 784 } 785 when (cfiIndex_bits_wen) { 786 cfiIndex_vec(r_idx).bits := r_offset 787 } 788 update_target(r_idx) := redirect.bits.cfiUpdate.target 789 if (isBackend) { 790 mispredict_vec(r_idx)(r_offset) := r_mispred 791 } 792 } 793 794 when(backendRedirectReg.valid && lastIsMispredict) { 795 updateCfiInfo(backendRedirectReg) 796 }.elsewhen (ifuRedirectToBpu.valid) { 797 updateCfiInfo(ifuRedirectToBpu, isBackend=false) 798 } 799 800 // *********************************************************************************** 801 // **************************** flush ptr and state queue **************************** 802 // *********************************************************************************** 803 804 val redirectVec = VecInit(backendRedirect, fromIfuRedirect) 805 806 // when redirect, we should reset ptrs and status queues 807 when(redirectVec.map(r => r.valid).reduce(_||_)){ 808 val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits))) 809 val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_) 810 val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level)) 811 val next = idx + 1.U 812 bpuPtr := next 813 ifuPtr := next 814 prefetchPtr := next 815 ifuWbPtr := next 816 when (notIfu) { 817 commitStateQueue(idx.value).zipWithIndex.foreach({ case (s, i) => 818 when(i.U > offset || i.U === offset && flushItSelf){ 819 s := c_invalid 820 } 821 }) 822 } 823 } 824 825 // only the valid bit is actually needed 826 io.toIfu.redirect.bits := backendRedirect.bits 827 io.toIfu.redirect.valid := stage2Flush 828 829 // commit 830 for (c <- io.fromBackend.rob_commits) { 831 when(c.valid) { 832 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited 833 // TODO: remove this 834 // For instruction fusions, we also update the next instruction 835 when (c.bits.commitType === 4.U) { 836 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited 837 }.elsewhen(c.bits.commitType === 5.U) { 838 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited 839 }.elsewhen(c.bits.commitType === 6.U) { 840 val index = (c.bits.ftqIdx + 1.U).value 841 commitStateQueue(index)(0) := c_commited 842 }.elsewhen(c.bits.commitType === 7.U) { 843 val index = (c.bits.ftqIdx + 1.U).value 844 commitStateQueue(index)(1) := c_commited 845 } 846 } 847 } 848 849 // **************************************************************** 850 // **************************** to bpu **************************** 851 // **************************************************************** 852 853 io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu) 854 855 val may_have_stall_from_bpu = RegInit(false.B) 856 val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu && 857 Cat(commitStateQueue(commPtr.value).map(s => { 858 s === c_invalid || s === c_commited 859 })).andR() 860 861 // commit reads 862 ftq_pc_mem.io.raddr.last := commPtr.value 863 val commit_pc_bundle = ftq_pc_mem.io.rdata.last 864 ftq_pd_mem.io.raddr.last := commPtr.value 865 val commit_pd = ftq_pd_mem.io.rdata.last 866 ftq_redirect_sram.io.ren.last := canCommit 867 ftq_redirect_sram.io.raddr.last := commPtr.value 868 val commit_spec_meta = ftq_redirect_sram.io.rdata.last 869 ftq_meta_1r_sram.io.ren(0) := canCommit 870 ftq_meta_1r_sram.io.raddr(0) := commPtr.value 871 val commit_meta = ftq_meta_1r_sram.io.rdata(0) 872 ftb_entry_mem.io.raddr.last := commPtr.value 873 val commit_ftb_entry = ftb_entry_mem.io.rdata.last 874 875 // need one cycle to read mem and srams 876 val do_commit_ptr = RegNext(commPtr) 877 val do_commit = RegNext(canCommit, init=false.B) 878 when (canCommit) { commPtr := commPtr + 1.U } 879 val commit_state = RegNext(commitStateQueue(commPtr.value)) 880 val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value)) 881 when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) { 882 can_commit_cfi.valid := false.B 883 } 884 val commit_cfi = RegNext(can_commit_cfi) 885 886 val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map { 887 case (mis, state) => mis && state === c_commited 888 }) 889 val can_commit_hit = entry_hit_status(commPtr.value) 890 val commit_hit = RegNext(can_commit_hit) 891 val commit_target = RegNext(update_target(commPtr.value)) 892 val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken 893 894 val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit 895 may_have_stall_from_bpu := can_commit_cfi.valid && !to_bpu_hit && !may_have_stall_from_bpu 896 897 io.toBpu.update := DontCare 898 io.toBpu.update.valid := commit_valid && do_commit 899 val update = io.toBpu.update.bits 900 update.false_hit := commit_hit === h_false_hit 901 update.pc := commit_pc_bundle.startAddr 902 update.meta := commit_meta.meta 903 update.full_target := commit_target 904 update.fromFtqRedirectSram(commit_spec_meta) 905 906 val commit_real_hit = commit_hit === h_hit 907 val update_ftb_entry = update.ftb_entry 908 909 val ftbEntryGen = Module(new FTBEntryGen).io 910 ftbEntryGen.start_addr := commit_pc_bundle.startAddr 911 ftbEntryGen.old_entry := commit_ftb_entry 912 ftbEntryGen.pd := commit_pd 913 ftbEntryGen.cfiIndex := commit_cfi 914 ftbEntryGen.target := commit_target 915 ftbEntryGen.hit := commit_real_hit 916 ftbEntryGen.mispredict_vec := commit_mispredict 917 918 update_ftb_entry := ftbEntryGen.new_entry 919 update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos 920 update.mispred_mask := ftbEntryGen.mispred_mask 921 update.old_entry := ftbEntryGen.is_old_entry 922 923 update.is_minimal := false.B 924 update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc) 925 update.full_pred.br_taken_mask := ftbEntryGen.taken_mask 926 update.full_pred.jalr_target := commit_target 927 update.full_pred.hit := true.B 928 when (update.full_pred.is_jalr) { 929 update.full_pred.targets.last := commit_target 930 } 931 932 // ****************************************************************************** 933 // **************************** commit perf counters **************************** 934 // ****************************************************************************** 935 936 val commit_inst_mask = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt 937 val commit_mispred_mask = commit_mispredict.asUInt 938 val commit_not_mispred_mask = ~commit_mispred_mask 939 940 val commit_br_mask = commit_pd.brMask.asUInt 941 val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W))) 942 val commit_cfi_mask = (commit_br_mask | commit_jmp_mask) 943 944 val mbpInstrs = commit_inst_mask & commit_cfi_mask 945 946 val mbpRights = mbpInstrs & commit_not_mispred_mask 947 val mbpWrongs = mbpInstrs & commit_mispred_mask 948 949 io.bpuInfo.bpRight := PopCount(mbpRights) 950 io.bpuInfo.bpWrong := PopCount(mbpWrongs) 951 952 // Cfi Info 953 for (i <- 0 until PredictWidth) { 954 val pc = commit_pc_bundle.startAddr + (i * instBytes).U 955 val v = commit_state(i) === c_commited 956 val isBr = commit_pd.brMask(i) 957 val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U 958 val isCfi = isBr || isJmp 959 val isTaken = commit_cfi.valid && commit_cfi.bits === i.U 960 val misPred = commit_mispredict(i) 961 // val ghist = commit_spec_meta.ghist.predHist 962 val histPtr = commit_spec_meta.histPtr 963 val predCycle = commit_meta.meta(63, 0) 964 val target = commit_target 965 966 val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}))) 967 val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_) 968 val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid)) 969 XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " + 970 p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " + 971 p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " + 972 p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n") 973 } 974 975 val enq = io.fromBpu.resp 976 val perf_redirect = io.fromBackend.redirect 977 978 XSPerfAccumulate("entry", validEntries) 979 XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready) 980 XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level) 981 XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)) 982 XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid) 983 984 XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid) 985 986 XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready) 987 XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn) 988 XSPerfAccumulate("bpu_to_ftq_bubble", bpuPtr === ifuPtr) 989 990 val from_bpu = io.fromBpu.resp.bits 991 def in_entry_len_map_gen(resp: BranchPredictionBundle)(stage: String) = { 992 assert(!resp.is_minimal) 993 val entry_len = (resp.ftb_entry.getFallThrough(resp.pc) - resp.pc) >> instOffsetBits 994 val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U) 995 val entry_len_map = (1 to PredictWidth+1).map(i => 996 f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.valid) 997 ).foldLeft(Map[String, UInt]())(_+_) 998 entry_len_map 999 } 1000 val s2_entry_len_map = in_entry_len_map_gen(from_bpu.s2)("s2") 1001 1002 val to_ifu = io.toIfu.req.bits 1003 1004 1005 1006 val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U) 1007 val commit_num_inst_map = (1 to PredictWidth).map(i => 1008 f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit) 1009 ).foldLeft(Map[String, UInt]())(_+_) 1010 1011 1012 1013 val commit_jal_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W))) 1014 val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W))) 1015 val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W))) 1016 val commit_ret_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W))) 1017 1018 1019 val mbpBRights = mbpRights & commit_br_mask 1020 val mbpJRights = mbpRights & commit_jal_mask 1021 val mbpIRights = mbpRights & commit_jalr_mask 1022 val mbpCRights = mbpRights & commit_call_mask 1023 val mbpRRights = mbpRights & commit_ret_mask 1024 1025 val mbpBWrongs = mbpWrongs & commit_br_mask 1026 val mbpJWrongs = mbpWrongs & commit_jal_mask 1027 val mbpIWrongs = mbpWrongs & commit_jalr_mask 1028 val mbpCWrongs = mbpWrongs & commit_call_mask 1029 val mbpRWrongs = mbpWrongs & commit_ret_mask 1030 1031 val commit_pred_stage = RegNext(pred_stage(commPtr.value)) 1032 1033 def pred_stage_map(src: UInt, name: String) = { 1034 (0 until numBpStages).map(i => 1035 f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i))) 1036 ).foldLeft(Map[String, UInt]())(_+_) 1037 } 1038 1039 val mispred_stage_map = pred_stage_map(mbpWrongs, "mispredict") 1040 val br_mispred_stage_map = pred_stage_map(mbpBWrongs, "br_mispredict") 1041 val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict") 1042 val correct_stage_map = pred_stage_map(mbpRights, "correct") 1043 val br_correct_stage_map = pred_stage_map(mbpBRights, "br_correct") 1044 val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct") 1045 1046 val update_valid = io.toBpu.update.valid 1047 def u(cond: Bool) = update_valid && cond 1048 val ftb_false_hit = u(update.false_hit) 1049 // assert(!ftb_false_hit) 1050 val ftb_hit = u(commit_hit === h_hit) 1051 1052 val ftb_new_entry = u(ftbEntryGen.is_init_entry) 1053 val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid 1054 val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0) 1055 val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid 1056 1057 val ftb_old_entry = u(ftbEntryGen.is_old_entry) 1058 1059 val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified) 1060 val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br) 1061 val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified) 1062 val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full 1063 val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified 1064 1065 val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits 1066 val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U) 1067 val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i => 1068 f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry) 1069 ).foldLeft(Map[String, UInt]())(_+_) 1070 val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i => 1071 f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry) 1072 ).foldLeft(Map[String, UInt]())(_+_) 1073 1074 val ftq_occupancy_map = (0 to FtqSize).map(i => 1075 f"ftq_has_entry_$i" ->( validEntries === i.U) 1076 ).foldLeft(Map[String, UInt]())(_+_) 1077 1078 val perfCountsMap = Map( 1079 "BpInstr" -> PopCount(mbpInstrs), 1080 "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs), 1081 "BpRight" -> PopCount(mbpRights), 1082 "BpWrong" -> PopCount(mbpWrongs), 1083 "BpBRight" -> PopCount(mbpBRights), 1084 "BpBWrong" -> PopCount(mbpBWrongs), 1085 "BpJRight" -> PopCount(mbpJRights), 1086 "BpJWrong" -> PopCount(mbpJWrongs), 1087 "BpIRight" -> PopCount(mbpIRights), 1088 "BpIWrong" -> PopCount(mbpIWrongs), 1089 "BpCRight" -> PopCount(mbpCRights), 1090 "BpCWrong" -> PopCount(mbpCWrongs), 1091 "BpRRight" -> PopCount(mbpRRights), 1092 "BpRWrong" -> PopCount(mbpRWrongs), 1093 1094 "ftb_false_hit" -> PopCount(ftb_false_hit), 1095 "ftb_hit" -> PopCount(ftb_hit), 1096 "ftb_new_entry" -> PopCount(ftb_new_entry), 1097 "ftb_new_entry_only_br" -> PopCount(ftb_new_entry_only_br), 1098 "ftb_new_entry_only_jmp" -> PopCount(ftb_new_entry_only_jmp), 1099 "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp), 1100 "ftb_old_entry" -> PopCount(ftb_old_entry), 1101 "ftb_modified_entry" -> PopCount(ftb_modified_entry), 1102 "ftb_modified_entry_new_br" -> PopCount(ftb_modified_entry_new_br), 1103 "ftb_jalr_target_modified" -> PopCount(ftb_modified_entry_jalr_target_modified), 1104 "ftb_modified_entry_br_full" -> PopCount(ftb_modified_entry_br_full), 1105 "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken) 1106 ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++ 1107 s2_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++ 1108 mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++ 1109 correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map 1110 1111 for((key, value) <- perfCountsMap) { 1112 XSPerfAccumulate(key, value) 1113 } 1114 1115 // --------------------------- Debug -------------------------------- 1116 // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable) 1117 XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable) 1118 XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n") 1119 XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n") 1120 XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " + 1121 p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n") 1122 XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n") 1123 1124 // def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1125 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1126 // case (((valid, pd), ans), taken) => 1127 // Mux(valid && pd.isBr, 1128 // isWrong ^ Mux(ans.hit.asBool, 1129 // Mux(ans.taken.asBool, taken && ans.target === commitEntry.target, 1130 // !taken), 1131 // !taken), 1132 // false.B) 1133 // } 1134 // } 1135 1136 // def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1137 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1138 // case (((valid, pd), ans), taken) => 1139 // Mux(valid && pd.isBr, 1140 // isWrong ^ Mux(ans.hit.asBool, 1141 // Mux(ans.taken.asBool, taken && ans.target === commitEntry.target, 1142 // !taken), 1143 // !taken), 1144 // false.B) 1145 // } 1146 // } 1147 1148 // def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1149 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1150 // case (((valid, pd), ans), taken) => 1151 // Mux(valid && pd.isBr, 1152 // isWrong ^ (ans.taken.asBool === taken), 1153 // false.B) 1154 // } 1155 // } 1156 1157 // def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1158 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1159 // case (((valid, pd), ans), taken) => 1160 // Mux(valid && (pd.isBr) && ans.hit.asBool, 1161 // isWrong ^ (!taken), 1162 // false.B) 1163 // } 1164 // } 1165 1166 // def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1167 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1168 // case (((valid, pd), ans), taken) => 1169 // Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool, 1170 // isWrong ^ (ans.target === commitEntry.target), 1171 // false.B) 1172 // } 1173 // } 1174 1175 // val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B) 1176 // val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B) 1177 // // btb and ubtb pred jal and jalr as well 1178 // val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B) 1179 // val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B) 1180 // val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B) 1181 // val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B) 1182 1183 // val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B) 1184 // val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B) 1185 1186 // val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B) 1187 // val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B) 1188 1189 val perfEvents = Seq( 1190 ("bpu_s2_redirect ", bpu_s2_redirect ), 1191 // ("bpu_s3_redirect ", bpu_s3_redirect ), 1192 ("bpu_to_ftq_stall ", enq.valid && ~enq.ready ), 1193 ("mispredictRedirect ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level), 1194 ("replayRedirect ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level) ), 1195 ("predecodeRedirect ", fromIfuRedirect.valid ), 1196 ("to_ifu_bubble ", io.toIfu.req.ready && !io.toIfu.req.valid ), 1197 ("from_bpu_real_bubble ", !enq.valid && enq.ready && allowBpuIn ), 1198 ("BpInstr ", PopCount(mbpInstrs) ), 1199 ("BpBInstr ", PopCount(mbpBRights | mbpBWrongs) ), 1200 ("BpRight ", PopCount(mbpRights) ), 1201 ("BpWrong ", PopCount(mbpWrongs) ), 1202 ("BpBRight ", PopCount(mbpBRights) ), 1203 ("BpBWrong ", PopCount(mbpBWrongs) ), 1204 ("BpJRight ", PopCount(mbpJRights) ), 1205 ("BpJWrong ", PopCount(mbpJWrongs) ), 1206 ("BpIRight ", PopCount(mbpIRights) ), 1207 ("BpIWrong ", PopCount(mbpIWrongs) ), 1208 ("BpCRight ", PopCount(mbpCRights) ), 1209 ("BpCWrong ", PopCount(mbpCWrongs) ), 1210 ("BpRRight ", PopCount(mbpRRights) ), 1211 ("BpRWrong ", PopCount(mbpRWrongs) ), 1212 ("ftb_false_hit ", PopCount(ftb_false_hit) ), 1213 ("ftb_hit ", PopCount(ftb_hit) ), 1214 ) 1215 generatePerfEvent() 1216} 1217