1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import xiangshan._ 24import xiangshan.backend.CtrlToFtqIO 25 26class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr]( 27 p => p(XSCoreParamsKey).FtqSize 28){ 29 override def cloneType = (new FtqPtr).asInstanceOf[this.type] 30} 31 32object FtqPtr { 33 def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = { 34 val ptr = Wire(new FtqPtr) 35 ptr.flag := f 36 ptr.value := v 37 ptr 38 } 39 def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = { 40 apply(!ptr.flag, ptr.value) 41 } 42} 43 44class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule { 45 46 val io = IO(new Bundle() { 47 val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W))) 48 val ren = Input(Vec(numRead, Bool())) 49 val rdata = Output(Vec(numRead, gen)) 50 val waddr = Input(UInt(log2Up(FtqSize).W)) 51 val wen = Input(Bool()) 52 val wdata = Input(gen) 53 }) 54 55 for(i <- 0 until numRead){ 56 val sram = Module(new SRAMTemplate(gen, FtqSize)) 57 sram.io.r.req.valid := io.ren(i) 58 sram.io.r.req.bits.setIdx := io.raddr(i) 59 io.rdata(i) := sram.io.r.resp.data(0) 60 sram.io.w.req.valid := io.wen 61 sram.io.w.req.bits.setIdx := io.waddr 62 sram.io.w.req.bits.data := VecInit(io.wdata) 63 } 64 65} 66 67class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils { 68 // TODO: move pftAddr, oversize, carry to another mem 69 val startAddr = UInt(VAddrBits.W) 70 val nextLineAddr = UInt(VAddrBits.W) 71 val isNextMask = Vec(PredictWidth, Bool()) 72 val oversize = Bool() 73 val fallThruError = Bool() 74 // val carry = Bool() 75 def getPc(offset: UInt) = { 76 def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1) 77 def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits) 78 Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)), 79 getOffset(startAddr)+offset, 0.U(instOffsetBits.W)) 80 } 81 def fromBranchPrediction(resp: BranchPredictionBundle) = { 82 def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1) 83 this.startAddr := resp.pc 84 this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U 85 this.isNextMask := VecInit((0 until PredictWidth).map(i => 86 (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool() 87 )) 88 this.oversize := resp.oversize 89 this.fallThruError := resp.fallThruError 90 this 91 } 92 override def toPrintable: Printable = { 93 p"startAddr:${Hexadecimal(startAddr)}" 94 } 95} 96 97class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle { 98 val brMask = Vec(PredictWidth, Bool()) 99 val jmpInfo = ValidUndirectioned(Vec(3, Bool())) 100 val jmpOffset = UInt(log2Ceil(PredictWidth).W) 101 val jalTarget = UInt(VAddrBits.W) 102 val rvcMask = Vec(PredictWidth, Bool()) 103 def hasJal = jmpInfo.valid && !jmpInfo.bits(0) 104 def hasJalr = jmpInfo.valid && jmpInfo.bits(0) 105 def hasCall = jmpInfo.valid && jmpInfo.bits(1) 106 def hasRet = jmpInfo.valid && jmpInfo.bits(2) 107 108 def fromPdWb(pdWb: PredecodeWritebackBundle) = { 109 val pds = pdWb.pd 110 this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid)) 111 this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR 112 this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid), 113 pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet))) 114 this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)) 115 this.rvcMask := VecInit(pds.map(pd => pd.isRVC)) 116 this.jalTarget := pdWb.jalTarget 117 } 118 119 def toPd(offset: UInt) = { 120 require(offset.getWidth == log2Ceil(PredictWidth)) 121 val pd = Wire(new PreDecodeInfo) 122 pd.valid := true.B 123 pd.isRVC := rvcMask(offset) 124 val isBr = brMask(offset) 125 val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0) 126 pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr) 127 pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1) 128 pd.isRet := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2) 129 pd 130 } 131} 132 133 134 135class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 136 val rasSp = UInt(log2Ceil(RasSize).W) 137 val rasEntry = new RASEntry 138 // val specCnt = Vec(numBr, UInt(10.W)) 139 // val ghist = new ShiftingGlobalHistory 140 val ghr = UInt(UbtbGHRLength.W) 141 val folded_hist = new AllFoldedHistories(foldedGHistInfos) 142 val histPtr = new CGHPtr 143 144 def fromBranchPrediction(resp: BranchPredictionBundle) = { 145 assert(!resp.is_minimal) 146 this.rasSp := resp.rasSp 147 this.rasEntry := resp.rasTop 148 this.ghr := resp.ghr 149 this.folded_hist := resp.folded_hist 150 this.histPtr := resp.histPtr 151 this 152 } 153} 154 155class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 156 val meta = UInt(MaxMetaLength.W) 157} 158 159class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle { 160 val target = UInt(VAddrBits.W) 161 val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)) 162} 163 164// class FtqEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 165// val startAddr = UInt(VAddrBits.W) 166// val fallThruAddr = UInt(VAddrBits.W) 167// val isNextMask = Vec(PredictWidth, Bool()) 168 169// val meta = UInt(MaxMetaLength.W) 170 171// val rasSp = UInt(log2Ceil(RasSize).W) 172// val rasEntry = new RASEntry 173// val hist = new ShiftingGlobalHistory 174// val specCnt = Vec(numBr, UInt(10.W)) 175 176// val valids = Vec(PredictWidth, Bool()) 177// val brMask = Vec(PredictWidth, Bool()) 178// // isJalr, isCall, isRet 179// val jmpInfo = ValidUndirectioned(Vec(3, Bool())) 180// val jmpOffset = UInt(log2Ceil(PredictWidth).W) 181 182// val mispredVec = Vec(PredictWidth, Bool()) 183// val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)) 184// val target = UInt(VAddrBits.W) 185// } 186 187class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle { 188 val ptr = Output(new FtqPtr) 189 val offset = Output(UInt(log2Ceil(PredictWidth).W)) 190 val data = Input(gen) 191 def apply(ptr: FtqPtr, offset: UInt) = { 192 this.ptr := ptr 193 this.offset := offset 194 this.data 195 } 196 override def cloneType = (new FtqRead(gen)).asInstanceOf[this.type] 197} 198 199 200class FtqToBpuIO(implicit p: Parameters) extends XSBundle { 201 val redirect = Valid(new BranchPredictionRedirect) 202 val update = Valid(new BranchPredictionUpdate) 203 val enq_ptr = Output(new FtqPtr) 204} 205 206class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper { 207 val req = Decoupled(new FetchRequestBundle) 208 val redirect = Valid(new Redirect) 209 val flushFromBpu = new Bundle { 210 // when ifu pipeline is not stalled, 211 // a packet from bpu s3 can reach f1 at most 212 val s2 = Valid(new FtqPtr) 213 // val s3 = Valid(new FtqPtr) 214 def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = { 215 src.valid && !isAfter(src.bits, idx_to_flush) 216 } 217 def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx) 218 // def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx) 219 } 220} 221 222trait HasBackendRedirectInfo extends HasXSParameter { 223 def numRedirect = exuParameters.JmpCnt + exuParameters.AluCnt + 1 224 def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself() 225} 226 227class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo { 228 val pc_reads = Vec(1 + numRedirect + 1 + 1, Flipped(new FtqRead(UInt(VAddrBits.W)))) 229 val target_read = Flipped(new FtqRead(UInt(VAddrBits.W))) 230 def getJumpPcRead = pc_reads.head 231 def getRedirectPcRead = VecInit(pc_reads.tail.dropRight(2)) 232 def getMemPredPcRead = pc_reads.init.last 233 def getRobFlushPcRead = pc_reads.last 234} 235 236 237class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter { 238 val io = IO(new Bundle { 239 val start_addr = Input(UInt(VAddrBits.W)) 240 val old_entry = Input(new FTBEntry) 241 val pd = Input(new Ftq_pd_Entry) 242 val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W))) 243 val target = Input(UInt(VAddrBits.W)) 244 val hit = Input(Bool()) 245 val mispredict_vec = Input(Vec(PredictWidth, Bool())) 246 247 val new_entry = Output(new FTBEntry) 248 val new_br_insert_pos = Output(Vec(numBr, Bool())) 249 val taken_mask = Output(Vec(numBr, Bool())) 250 val mispred_mask = Output(Vec(numBr+1, Bool())) 251 252 // for perf counters 253 val is_init_entry = Output(Bool()) 254 val is_old_entry = Output(Bool()) 255 val is_new_br = Output(Bool()) 256 val is_jalr_target_modified = Output(Bool()) 257 val is_always_taken_modified = Output(Bool()) 258 val is_br_full = Output(Bool()) 259 }) 260 261 // no mispredictions detected at predecode 262 val hit = io.hit 263 val pd = io.pd 264 265 val init_entry = WireInit(0.U.asTypeOf(new FTBEntry)) 266 267 268 val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid 269 val entry_has_jmp = pd.jmpInfo.valid 270 val new_jmp_is_jal = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid 271 val new_jmp_is_jalr = entry_has_jmp && pd.jmpInfo.bits(0) && io.cfiIndex.valid 272 val new_jmp_is_call = entry_has_jmp && pd.jmpInfo.bits(1) && io.cfiIndex.valid 273 val new_jmp_is_ret = entry_has_jmp && pd.jmpInfo.bits(2) && io.cfiIndex.valid 274 val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last 275 val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last 276 277 val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal 278 val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr 279 280 def carryPos = log2Ceil(PredictWidth)+instOffsetBits+1 281 def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits) 282 // if not hit, establish a new entry 283 init_entry.valid := true.B 284 // tag is left for ftb to assign 285 286 // case br 287 val init_br_slot = init_entry.getSlotForBr(0) 288 when (cfi_is_br) { 289 init_br_slot.valid := true.B 290 init_br_slot.offset := io.cfiIndex.bits 291 init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1) 292 init_entry.always_taken(0) := true.B // set to always taken on init 293 } 294 295 // case jmp 296 when (entry_has_jmp) { 297 init_entry.tailSlot.offset := pd.jmpOffset 298 init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr 299 init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false) 300 } 301 302 val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U) 303 init_entry.pftAddr := Mux(entry_has_jmp, jmpPft, getLower(io.start_addr) + ((FetchWidth*4)>>instOffsetBits).U + Mux(last_br_rvi, 1.U, 0.U)) 304 init_entry.carry := Mux(entry_has_jmp, jmpPft(carryPos-instOffsetBits), io.start_addr(carryPos-1) || (io.start_addr(carryPos-2, instOffsetBits).andR && last_br_rvi)) 305 init_entry.isJalr := new_jmp_is_jalr 306 init_entry.isCall := new_jmp_is_call 307 init_entry.isRet := new_jmp_is_ret 308 init_entry.last_is_rvc := Mux(entry_has_jmp, pd.rvcMask(pd.jmpOffset), pd.rvcMask.last) 309 310 init_entry.oversize := last_br_rvi || last_jmp_rvi 311 312 // if hit, check whether a new cfi(only br is possible) is detected 313 val oe = io.old_entry 314 val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits) 315 val br_recorded = br_recorded_vec.asUInt.orR 316 val is_new_br = cfi_is_br && !br_recorded 317 val new_br_offset = io.cfiIndex.bits 318 // vec(i) means new br will be inserted BEFORE old br(i) 319 val allBrSlotsVec = oe.allSlotsForBr 320 val new_br_insert_onehot = VecInit((0 until numBr).map{ 321 i => i match { 322 case 0 => 323 !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset 324 case idx => 325 allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset && 326 (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset) 327 } 328 }) 329 330 val old_entry_modified = WireInit(io.old_entry) 331 for (i <- 0 until numBr) { 332 val slot = old_entry_modified.allSlotsForBr(i) 333 when (new_br_insert_onehot(i)) { 334 slot.valid := true.B 335 slot.offset := new_br_offset 336 slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1) 337 old_entry_modified.always_taken(i) := true.B 338 }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) { 339 old_entry_modified.always_taken(i) := false.B 340 // all other fields remain unchanged 341 }.otherwise { 342 // case i == 0, remain unchanged 343 if (i != 0) { 344 val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid 345 when (!noNeedToMoveFromFormerSlot) { 346 slot.fromAnotherSlot(oe.allSlotsForBr(i-1)) 347 old_entry_modified.always_taken(i) := oe.always_taken(i) 348 } 349 } 350 } 351 } 352 353 // two circumstances: 354 // 1. oe: | br | j |, new br should be in front of j, thus addr of j should be new pft 355 // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either 356 // the previous last br or the new br 357 val may_have_to_replace = oe.noEmptySlotForNewBr 358 val pft_need_to_change = is_new_br && may_have_to_replace 359 // it should either be the given last br or the new br 360 when (pft_need_to_change) { 361 val new_pft_offset = 362 Mux(!new_br_insert_onehot.asUInt.orR, 363 new_br_offset, oe.allSlotsForBr.last.offset) 364 365 // set jmp to invalid 366 old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset 367 old_entry_modified.last_is_rvc := pd.rvcMask(new_pft_offset - 1.U) // TODO: fix this 368 old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool 369 old_entry_modified.oversize := false.B 370 old_entry_modified.isCall := false.B 371 old_entry_modified.isRet := false.B 372 old_entry_modified.isJalr := false.B 373 } 374 375 val old_entry_jmp_target_modified = WireInit(oe) 376 val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits 377 val old_tail_is_jmp = !oe.tailSlot.sharing 378 val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target 379 when (jalr_target_modified) { 380 old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target) 381 old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool())) 382 } 383 384 val old_entry_always_taken = WireInit(oe) 385 val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not 386 for (i <- 0 until numBr) { 387 old_entry_always_taken.always_taken(i) := 388 oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i) 389 always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i) 390 } 391 val always_taken_modified = always_taken_modified_vec.reduce(_||_) 392 393 394 395 val derived_from_old_entry = 396 Mux(is_new_br, old_entry_modified, 397 Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken)) 398 399 400 io.new_entry := Mux(!hit, init_entry, derived_from_old_entry) 401 402 io.new_br_insert_pos := new_br_insert_onehot 403 io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{ 404 case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v 405 }) 406 for (i <- 0 until numBr) { 407 io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i)) 408 } 409 io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset) 410 411 // for perf counters 412 io.is_init_entry := !hit 413 io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified 414 io.is_new_br := hit && is_new_br 415 io.is_jalr_target_modified := hit && jalr_target_modified 416 io.is_always_taken_modified := hit && always_taken_modified 417 io.is_br_full := hit && is_new_br && may_have_to_replace 418} 419 420class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper 421 with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents { 422 val io = IO(new Bundle { 423 val fromBpu = Flipped(new BpuToFtqIO) 424 val fromIfu = Flipped(new IfuToFtqIO) 425 val fromBackend = Flipped(new CtrlToFtqIO) 426 427 val toBpu = new FtqToBpuIO 428 val toIfu = new FtqToIfuIO 429 val toBackend = new FtqToCtrlIO 430 431 val bpuInfo = new Bundle { 432 val bpRight = Output(UInt(XLEN.W)) 433 val bpWrong = Output(UInt(XLEN.W)) 434 } 435 }) 436 io.bpuInfo := DontCare 437 438 val stage2Redirect = io.fromBackend.stage2Redirect 439 val stage3Redirect = RegNext(io.fromBackend.stage2Redirect) 440 441 val stage2Flush = stage2Redirect.valid 442 val backendFlush = stage2Flush || RegNext(stage2Flush) 443 val ifuFlush = Wire(Bool()) 444 445 val flush = stage2Flush || RegNext(stage2Flush) 446 447 val allowBpuIn, allowToIfu = WireInit(false.B) 448 val flushToIfu = !allowToIfu 449 allowBpuIn := !ifuFlush && !stage2Redirect.valid && !stage3Redirect.valid 450 allowToIfu := !ifuFlush && !stage2Redirect.valid && !stage3Redirect.valid 451 452 val bpuPtr, ifuPtr, ifuWbPtr, commPtr = RegInit(FtqPtr(false.B, 0.U)) 453 val validEntries = distanceBetween(bpuPtr, commPtr) 454 455 // ********************************************************************** 456 // **************************** enq from bpu **************************** 457 // ********************************************************************** 458 val new_entry_ready = validEntries < FtqSize.U 459 io.fromBpu.resp.ready := new_entry_ready 460 461 val bpu_s2_resp = io.fromBpu.resp.bits.s2 462 // val bpu_s3_resp = io.fromBpu.resp.bits.s3 463 val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect 464 // val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect 465 466 io.toBpu.enq_ptr := bpuPtr 467 val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1 468 val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect/* || bpu_s3_redirect */) && allowBpuIn 469 470 val bpu_in_resp = io.fromBpu.resp.bits.selectedResp 471 val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx 472 val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx) 473 val bpu_in_resp_idx = bpu_in_resp_ptr.value 474 475 // read ports: jumpPc + redirects + loadPred + robFlush + ifuReq1 + ifuReq2 + commitUpdate 476 val ftq_pc_mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize, 1+numRedirect+2+1+1+1, 1)) 477 // resp from uBTB 478 ftq_pc_mem.io.wen(0) := bpu_in_fire 479 ftq_pc_mem.io.waddr(0) := bpu_in_resp_idx 480 ftq_pc_mem.io.wdata(0).fromBranchPrediction(bpu_in_resp) 481 482 // ifuRedirect + backendRedirect + commit 483 val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1)) 484 // these info is intended to enq at the last stage of bpu 485 ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid 486 ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value 487 ftq_redirect_sram.io.wdata.fromBranchPrediction(io.fromBpu.resp.bits.lastStage) 488 489 val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1)) 490 // these info is intended to enq at the last stage of bpu 491 ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid 492 ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value 493 ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.meta 494 // ifuRedirect + backendRedirect + commit 495 val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1)) 496 ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid 497 ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value 498 ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.lastStage.ftb_entry 499 500 501 // multi-write 502 val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough 503 val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)))) 504 val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool()))) 505 val pred_stage = Reg(Vec(FtqSize, UInt(2.W))) 506 507 val c_invalid :: c_valid :: c_commited :: Nil = Enum(3) 508 val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) { 509 VecInit(Seq.fill(PredictWidth)(c_invalid)) 510 })) 511 512 val f_to_send :: f_sent :: Nil = Enum(2) 513 val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent))) 514 515 val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3) 516 val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit))) 517 518 519 when (bpu_in_fire) { 520 entry_fetch_status(bpu_in_resp_idx) := f_to_send 521 commitStateQueue(bpu_in_resp_idx) := VecInit(Seq.fill(PredictWidth)(c_invalid)) 522 cfiIndex_vec(bpu_in_resp_idx) := bpu_in_resp.cfiIndex 523 mispredict_vec(bpu_in_resp_idx) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B))) 524 update_target(bpu_in_resp_idx) := bpu_in_resp.getTarget 525 pred_stage(bpu_in_resp_idx) := bpu_in_stage 526 } 527 528 bpuPtr := bpuPtr + enq_fire 529 ifuPtr := ifuPtr + io.toIfu.req.fire 530 531 // only use ftb result to assign hit status 532 when (bpu_s2_resp.valid) { 533 entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit) 534 } 535 536 537 io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect 538 io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx 539 when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) { 540 bpuPtr := bpu_s2_resp.ftq_idx + 1.U 541 // only when ifuPtr runs ahead of bpu s2 resp should we recover it 542 when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) { 543 ifuPtr := bpu_s2_resp.ftq_idx 544 } 545 } 546 547 // io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect 548 // io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx 549 // when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) { 550 // bpuPtr := bpu_s3_resp.ftq_idx + 1.U 551 // // only when ifuPtr runs ahead of bpu s2 resp should we recover it 552 // when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) { 553 // ifuPtr := bpu_s3_resp.ftq_idx 554 // } 555 // XSError(true.B, "\ns3_redirect mechanism not implemented!\n") 556 // } 557 558 XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n") 559 560 // **************************************************************** 561 // **************************** to ifu **************************** 562 // **************************************************************** 563 val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata(0), enable=bpu_in_fire) 564 val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr) 565 val last_cycle_bpu_in = RegNext(bpu_in_fire) 566 val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire) 567 568 // read pc and target 569 ftq_pc_mem.io.raddr.init.init.last := ifuPtr.value 570 ftq_pc_mem.io.raddr.init.last := (ifuPtr+1.U).value 571 572 io.toIfu.req.valid := allowToIfu && entry_fetch_status(ifuPtr.value) === f_to_send && ifuPtr =/= bpuPtr 573 io.toIfu.req.bits.ftqIdx := ifuPtr 574 io.toIfu.req.bits.nextStartAddr := update_target(ifuPtr.value) 575 io.toIfu.req.bits.ftqOffset := cfiIndex_vec(ifuPtr.value) 576 577 val toIfuPcBundle = Wire(new Ftq_RF_Components) 578 when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) { 579 toIfuPcBundle := bpu_in_bypass_buf 580 }.elsewhen (last_cycle_to_ifu_fire) { 581 toIfuPcBundle := ftq_pc_mem.io.rdata.init.last 582 }.otherwise { 583 toIfuPcBundle := ftq_pc_mem.io.rdata.init.init.last 584 } 585 586 io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle) 587 588 // when fall through is smaller in value than start address, there must be a false hit 589 when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) { 590 when (io.toIfu.req.fire && 591 !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr)/* && 592 !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr) */ 593 ) { 594 entry_hit_status(ifuPtr.value) := h_false_hit 595 XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr) 596 } 597 XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr) 598 } 599 600 val ifu_req_should_be_flushed = 601 io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx)/* || 602 io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx) */ 603 604 when (io.toIfu.req.fire && !ifu_req_should_be_flushed) { 605 entry_fetch_status(ifuPtr.value) := f_sent 606 } 607 608 609 // ********************************************************************* 610 // **************************** wb from ifu **************************** 611 // ********************************************************************* 612 val pdWb = io.fromIfu.pdWb 613 val pds = pdWb.bits.pd 614 val ifu_wb_valid = pdWb.valid 615 val ifu_wb_idx = pdWb.bits.ftqIdx.value 616 // read ports: commit update 617 val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1)) 618 ftq_pd_mem.io.wen(0) := ifu_wb_valid 619 ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value 620 ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits) 621 622 val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid 623 val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid 624 val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B) 625 val pd_reg = RegEnable(pds, enable = pdWb.valid) 626 val start_pc_reg = RegEnable(pdWb.bits.pc(0), enable = pdWb.valid) 627 val wb_idx_reg = RegEnable(ifu_wb_idx, enable = pdWb.valid) 628 629 when (ifu_wb_valid) { 630 val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{ 631 case (v, inRange) => v && inRange 632 }) 633 (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{ 634 case (qe, v) => when (v) { qe := c_valid } 635 } 636 } 637 638 ifuWbPtr := ifuWbPtr + ifu_wb_valid 639 640 ftb_entry_mem.io.raddr.head := ifu_wb_idx 641 val has_false_hit = WireInit(false.B) 642 when (RegNext(hit_pd_valid)) { 643 // check for false hit 644 val pred_ftb_entry = ftb_entry_mem.io.rdata.head 645 val brSlots = pred_ftb_entry.brSlots 646 val tailSlot = pred_ftb_entry.tailSlot 647 // we check cfis that bpu predicted 648 649 // bpu predicted branches but denied by predecode 650 val br_false_hit = 651 brSlots.map{ 652 s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr) 653 }.reduce(_||_) || 654 (tailSlot.valid && pred_ftb_entry.tailSlot.sharing && 655 !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr)) 656 657 val jmpOffset = tailSlot.offset 658 val jmp_pd = pd_reg(jmpOffset) 659 val jal_false_hit = pred_ftb_entry.jmpValid && 660 ((pred_ftb_entry.isJal && !(jmp_pd.valid && jmp_pd.isJal)) || 661 (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) || 662 (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) || 663 (pred_ftb_entry.isRet && !(jmp_pd.valid && jmp_pd.isRet)) 664 ) 665 666 has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg 667 XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0)) 668 669 assert(!has_false_hit) 670 } 671 672 when (has_false_hit) { 673 entry_hit_status(wb_idx_reg) := h_false_hit 674 } 675 676 677 // ********************************************************************** 678 // **************************** backend read **************************** 679 // ********************************************************************** 680 681 // pc reads 682 for ((req, i) <- io.toBackend.pc_reads.zipWithIndex) { 683 ftq_pc_mem.io.raddr(i) := req.ptr.value 684 req.data := ftq_pc_mem.io.rdata(i).getPc(RegNext(req.offset)) 685 } 686 // target read 687 io.toBackend.target_read.data := RegNext(update_target(io.toBackend.target_read.ptr.value)) 688 689 // ******************************************************************************* 690 // **************************** redirect from backend **************************** 691 // ******************************************************************************* 692 693 // redirect read cfiInfo, couples to redirectGen s2 694 ftq_redirect_sram.io.ren.init.last := io.fromBackend.stage2Redirect.valid 695 ftq_redirect_sram.io.raddr.init.last := io.fromBackend.stage2Redirect.bits.ftqIdx.value 696 697 ftb_entry_mem.io.raddr.init.last := io.fromBackend.stage2Redirect.bits.ftqIdx.value 698 699 val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last 700 val fromBackendRedirect = WireInit(stage3Redirect) 701 val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate 702 backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo) 703 704 val r_ftb_entry = ftb_entry_mem.io.rdata.init.last 705 val r_ftqOffset = fromBackendRedirect.bits.ftqOffset 706 707 when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) { 708 backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +& 709 (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) && 710 !r_ftb_entry.newBrCanNotInsert(r_ftqOffset)) 711 712 backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) || 713 !r_ftb_entry.newBrCanNotInsert(r_ftqOffset)) 714 }.otherwise { 715 backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt 716 backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt 717 } 718 719 720 // *************************************************************************** 721 // **************************** redirect from ifu **************************** 722 // *************************************************************************** 723 val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect))) 724 fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush 725 fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx 726 fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits 727 fromIfuRedirect.bits.level := RedirectLevel.flushAfter 728 729 val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate 730 ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits) 731 ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits) 732 ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid 733 ifuRedirectCfiUpdate.target := pdWb.bits.target 734 ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid 735 ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid 736 737 val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect))) 738 val ifuRedirectToBpu = WireInit(ifuRedirectReg) 739 ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid 740 741 ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid 742 ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value 743 744 ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value 745 746 val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate 747 toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head) 748 when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) { 749 toBpuCfi.target := toBpuCfi.rasEntry.retAddr 750 } 751 752 // ********************************************************************* 753 // **************************** wb from exu **************************** 754 // ********************************************************************* 755 756 def extractRedirectInfo(wb: Valid[Redirect]) = { 757 val ftqIdx = wb.bits.ftqIdx.value 758 val ftqOffset = wb.bits.ftqOffset 759 val taken = wb.bits.cfiUpdate.taken 760 val mispred = wb.bits.cfiUpdate.isMisPred 761 (wb.valid, ftqIdx, ftqOffset, taken, mispred) 762 } 763 764 // fix mispredict entry 765 val lastIsMispredict = RegNext( 766 stage2Redirect.valid && stage2Redirect.bits.level === RedirectLevel.flushAfter, init = false.B 767 ) 768 769 def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = { 770 val (r_valid, r_idx, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect) 771 val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits 772 val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits 773 when (cfiIndex_bits_wen || cfiIndex_valid_wen) { 774 cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken 775 } 776 when (cfiIndex_bits_wen) { 777 cfiIndex_vec(r_idx).bits := r_offset 778 } 779 update_target(r_idx) := redirect.bits.cfiUpdate.target 780 if (isBackend) { 781 mispredict_vec(r_idx)(r_offset) := r_mispred 782 } 783 } 784 785 when(stage3Redirect.valid && lastIsMispredict) { 786 updateCfiInfo(stage3Redirect) 787 }.elsewhen (ifuRedirectToBpu.valid) { 788 updateCfiInfo(ifuRedirectToBpu, isBackend=false) 789 } 790 791 // *********************************************************************************** 792 // **************************** flush ptr and state queue **************************** 793 // *********************************************************************************** 794 795 val redirectVec = VecInit(stage2Redirect, fromIfuRedirect) 796 797 // when redirect, we should reset ptrs and status queues 798 when(redirectVec.map(r => r.valid).reduce(_||_)){ 799 val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits))) 800 val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_) 801 val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level)) 802 val next = idx + 1.U 803 bpuPtr := next 804 ifuPtr := next 805 ifuWbPtr := next 806 when (notIfu) { 807 commitStateQueue(idx.value).zipWithIndex.foreach({ case (s, i) => 808 when(i.U > offset || i.U === offset && flushItSelf){ 809 s := c_invalid 810 } 811 }) 812 } 813 } 814 815 // only the valid bit is actually needed 816 io.toIfu.redirect.bits := stage2Redirect.bits 817 io.toIfu.redirect.valid := stage2Flush 818 819 // commit 820 for (c <- io.fromBackend.rob_commits) { 821 when(c.valid) { 822 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited 823 // TODO: remove this 824 // For instruction fusions, we also update the next instruction 825 when (c.bits.commitType === 4.U) { 826 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited 827 }.elsewhen(c.bits.commitType === 5.U) { 828 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited 829 }.elsewhen(c.bits.commitType === 6.U) { 830 val index = (c.bits.ftqIdx + 1.U).value 831 commitStateQueue(index)(0) := c_commited 832 }.elsewhen(c.bits.commitType === 7.U) { 833 val index = (c.bits.ftqIdx + 1.U).value 834 commitStateQueue(index)(1) := c_commited 835 } 836 } 837 } 838 839 // **************************************************************** 840 // **************************** to bpu **************************** 841 // **************************************************************** 842 843 io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu) 844 845 val may_have_stall_from_bpu = RegInit(false.B) 846 val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu && 847 Cat(commitStateQueue(commPtr.value).map(s => { 848 s === c_invalid || s === c_commited 849 })).andR() 850 851 // commit reads 852 ftq_pc_mem.io.raddr.last := commPtr.value 853 val commit_pc_bundle = ftq_pc_mem.io.rdata.last 854 ftq_pd_mem.io.raddr.last := commPtr.value 855 val commit_pd = ftq_pd_mem.io.rdata.last 856 ftq_redirect_sram.io.ren.last := canCommit 857 ftq_redirect_sram.io.raddr.last := commPtr.value 858 val commit_spec_meta = ftq_redirect_sram.io.rdata.last 859 ftq_meta_1r_sram.io.ren(0) := canCommit 860 ftq_meta_1r_sram.io.raddr(0) := commPtr.value 861 val commit_meta = ftq_meta_1r_sram.io.rdata(0) 862 ftb_entry_mem.io.raddr.last := commPtr.value 863 val commit_ftb_entry = ftb_entry_mem.io.rdata.last 864 865 // need one cycle to read mem and srams 866 val do_commit_ptr = RegNext(commPtr) 867 val do_commit = RegNext(canCommit, init=false.B) 868 when (canCommit) { commPtr := commPtr + 1.U } 869 val commit_state = RegNext(commitStateQueue(commPtr.value)) 870 val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value)) 871 when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) { 872 can_commit_cfi.valid := false.B 873 } 874 val commit_cfi = RegNext(can_commit_cfi) 875 876 val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map { 877 case (mis, state) => mis && state === c_commited 878 }) 879 val can_commit_hit = entry_hit_status(commPtr.value) 880 val commit_hit = RegNext(can_commit_hit) 881 val commit_target = RegNext(update_target(commPtr.value)) 882 val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken 883 884 val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit 885 may_have_stall_from_bpu := can_commit_cfi.valid && !to_bpu_hit && !may_have_stall_from_bpu 886 887 io.toBpu.update := DontCare 888 io.toBpu.update.valid := commit_valid && do_commit 889 val update = io.toBpu.update.bits 890 update.false_hit := commit_hit === h_false_hit 891 update.pc := commit_pc_bundle.startAddr 892 update.meta := commit_meta.meta 893 update.full_target := commit_target 894 update.fromFtqRedirectSram(commit_spec_meta) 895 896 val commit_real_hit = commit_hit === h_hit 897 val update_ftb_entry = update.ftb_entry 898 899 val ftbEntryGen = Module(new FTBEntryGen).io 900 ftbEntryGen.start_addr := commit_pc_bundle.startAddr 901 ftbEntryGen.old_entry := commit_ftb_entry 902 ftbEntryGen.pd := commit_pd 903 ftbEntryGen.cfiIndex := commit_cfi 904 ftbEntryGen.target := commit_target 905 ftbEntryGen.hit := commit_real_hit 906 ftbEntryGen.mispredict_vec := commit_mispredict 907 908 update_ftb_entry := ftbEntryGen.new_entry 909 update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos 910 update.mispred_mask := ftbEntryGen.mispred_mask 911 update.old_entry := ftbEntryGen.is_old_entry 912 913 update.is_minimal := false.B 914 update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc) 915 update.full_pred.br_taken_mask := ftbEntryGen.taken_mask 916 update.full_pred.jalr_target := commit_target 917 update.full_pred.hit := true.B 918 when (update.full_pred.is_jalr) { 919 update.full_pred.targets.last := commit_target 920 } 921 922 // ****************************************************************************** 923 // **************************** commit perf counters **************************** 924 // ****************************************************************************** 925 926 val commit_inst_mask = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt 927 val commit_mispred_mask = commit_mispredict.asUInt 928 val commit_not_mispred_mask = ~commit_mispred_mask 929 930 val commit_br_mask = commit_pd.brMask.asUInt 931 val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W))) 932 val commit_cfi_mask = (commit_br_mask | commit_jmp_mask) 933 934 val mbpInstrs = commit_inst_mask & commit_cfi_mask 935 936 val mbpRights = mbpInstrs & commit_not_mispred_mask 937 val mbpWrongs = mbpInstrs & commit_mispred_mask 938 939 io.bpuInfo.bpRight := PopCount(mbpRights) 940 io.bpuInfo.bpWrong := PopCount(mbpWrongs) 941 942 // Cfi Info 943 for (i <- 0 until PredictWidth) { 944 val pc = commit_pc_bundle.startAddr + (i * instBytes).U 945 val v = commit_state(i) === c_commited 946 val isBr = commit_pd.brMask(i) 947 val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U 948 val isCfi = isBr || isJmp 949 val isTaken = commit_cfi.valid && commit_cfi.bits === i.U 950 val misPred = commit_mispredict(i) 951 // val ghist = commit_spec_meta.ghist.predHist 952 val histPtr = commit_spec_meta.histPtr 953 val predCycle = commit_meta.meta(63, 0) 954 val target = commit_target 955 956 val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}))) 957 val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_) 958 val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid)) 959 XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " + 960 p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " + 961 p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " + 962 p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n") 963 } 964 965 val enq = io.fromBpu.resp 966 val perf_redirect = io.fromBackend.stage2Redirect 967 968 XSPerfAccumulate("entry", validEntries) 969 XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready) 970 XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level) 971 XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)) 972 XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid) 973 974 XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid) 975 976 XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready) 977 XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn) 978 XSPerfAccumulate("bpu_to_ftq_bubble", bpuPtr === ifuPtr) 979 980 val from_bpu = io.fromBpu.resp.bits 981 def in_entry_len_map_gen(resp: BranchPredictionBundle)(stage: String) = { 982 assert(!resp.is_minimal) 983 val entry_len = (resp.ftb_entry.getFallThrough(resp.pc) - resp.pc) >> instOffsetBits 984 val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U) 985 val entry_len_map = (1 to PredictWidth+1).map(i => 986 f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.valid) 987 ).foldLeft(Map[String, UInt]())(_+_) 988 entry_len_map 989 } 990 val s2_entry_len_map = in_entry_len_map_gen(from_bpu.s2)("s2") 991 992 val to_ifu = io.toIfu.req.bits 993 994 995 996 val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U) 997 val commit_num_inst_map = (1 to PredictWidth).map(i => 998 f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit) 999 ).foldLeft(Map[String, UInt]())(_+_) 1000 1001 1002 1003 val commit_jal_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W))) 1004 val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W))) 1005 val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W))) 1006 val commit_ret_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W))) 1007 1008 1009 val mbpBRights = mbpRights & commit_br_mask 1010 val mbpJRights = mbpRights & commit_jal_mask 1011 val mbpIRights = mbpRights & commit_jalr_mask 1012 val mbpCRights = mbpRights & commit_call_mask 1013 val mbpRRights = mbpRights & commit_ret_mask 1014 1015 val mbpBWrongs = mbpWrongs & commit_br_mask 1016 val mbpJWrongs = mbpWrongs & commit_jal_mask 1017 val mbpIWrongs = mbpWrongs & commit_jalr_mask 1018 val mbpCWrongs = mbpWrongs & commit_call_mask 1019 val mbpRWrongs = mbpWrongs & commit_ret_mask 1020 1021 val commit_pred_stage = RegNext(pred_stage(commPtr.value)) 1022 1023 def pred_stage_map(src: UInt, name: String) = { 1024 (0 until numBpStages).map(i => 1025 f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i))) 1026 ).foldLeft(Map[String, UInt]())(_+_) 1027 } 1028 1029 val mispred_stage_map = pred_stage_map(mbpWrongs, "mispredict") 1030 val br_mispred_stage_map = pred_stage_map(mbpBWrongs, "br_mispredict") 1031 val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict") 1032 val correct_stage_map = pred_stage_map(mbpRights, "correct") 1033 val br_correct_stage_map = pred_stage_map(mbpBRights, "br_correct") 1034 val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct") 1035 1036 val update_valid = io.toBpu.update.valid 1037 def u(cond: Bool) = update_valid && cond 1038 val ftb_false_hit = u(update.false_hit) 1039 // assert(!ftb_false_hit) 1040 val ftb_hit = u(commit_hit === h_hit) 1041 1042 val ftb_new_entry = u(ftbEntryGen.is_init_entry) 1043 val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid 1044 val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0) 1045 val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid 1046 1047 val ftb_old_entry = u(ftbEntryGen.is_old_entry) 1048 1049 val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified) 1050 val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br) 1051 val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified) 1052 val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full 1053 val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified 1054 1055 val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits 1056 val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U) 1057 val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i => 1058 f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry) 1059 ).foldLeft(Map[String, UInt]())(_+_) 1060 val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i => 1061 f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry) 1062 ).foldLeft(Map[String, UInt]())(_+_) 1063 1064 val ftq_occupancy_map = (0 to FtqSize).map(i => 1065 f"ftq_has_entry_$i" ->( validEntries === i.U) 1066 ).foldLeft(Map[String, UInt]())(_+_) 1067 1068 val perfCountsMap = Map( 1069 "BpInstr" -> PopCount(mbpInstrs), 1070 "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs), 1071 "BpRight" -> PopCount(mbpRights), 1072 "BpWrong" -> PopCount(mbpWrongs), 1073 "BpBRight" -> PopCount(mbpBRights), 1074 "BpBWrong" -> PopCount(mbpBWrongs), 1075 "BpJRight" -> PopCount(mbpJRights), 1076 "BpJWrong" -> PopCount(mbpJWrongs), 1077 "BpIRight" -> PopCount(mbpIRights), 1078 "BpIWrong" -> PopCount(mbpIWrongs), 1079 "BpCRight" -> PopCount(mbpCRights), 1080 "BpCWrong" -> PopCount(mbpCWrongs), 1081 "BpRRight" -> PopCount(mbpRRights), 1082 "BpRWrong" -> PopCount(mbpRWrongs), 1083 1084 "ftb_false_hit" -> PopCount(ftb_false_hit), 1085 "ftb_hit" -> PopCount(ftb_hit), 1086 "ftb_new_entry" -> PopCount(ftb_new_entry), 1087 "ftb_new_entry_only_br" -> PopCount(ftb_new_entry_only_br), 1088 "ftb_new_entry_only_jmp" -> PopCount(ftb_new_entry_only_jmp), 1089 "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp), 1090 "ftb_old_entry" -> PopCount(ftb_old_entry), 1091 "ftb_modified_entry" -> PopCount(ftb_modified_entry), 1092 "ftb_modified_entry_new_br" -> PopCount(ftb_modified_entry_new_br), 1093 "ftb_jalr_target_modified" -> PopCount(ftb_modified_entry_jalr_target_modified), 1094 "ftb_modified_entry_br_full" -> PopCount(ftb_modified_entry_br_full), 1095 "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken) 1096 ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++ 1097 s2_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++ 1098 mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++ 1099 correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map 1100 1101 for((key, value) <- perfCountsMap) { 1102 XSPerfAccumulate(key, value) 1103 } 1104 1105 // --------------------------- Debug -------------------------------- 1106 // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable) 1107 XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable) 1108 XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n") 1109 XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n") 1110 XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " + 1111 p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n") 1112 XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n") 1113 1114 // def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1115 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1116 // case (((valid, pd), ans), taken) => 1117 // Mux(valid && pd.isBr, 1118 // isWrong ^ Mux(ans.hit.asBool, 1119 // Mux(ans.taken.asBool, taken && ans.target === commitEntry.target, 1120 // !taken), 1121 // !taken), 1122 // false.B) 1123 // } 1124 // } 1125 1126 // def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1127 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1128 // case (((valid, pd), ans), taken) => 1129 // Mux(valid && pd.isBr, 1130 // isWrong ^ Mux(ans.hit.asBool, 1131 // Mux(ans.taken.asBool, taken && ans.target === commitEntry.target, 1132 // !taken), 1133 // !taken), 1134 // false.B) 1135 // } 1136 // } 1137 1138 // def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1139 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1140 // case (((valid, pd), ans), taken) => 1141 // Mux(valid && pd.isBr, 1142 // isWrong ^ (ans.taken.asBool === taken), 1143 // false.B) 1144 // } 1145 // } 1146 1147 // def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1148 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1149 // case (((valid, pd), ans), taken) => 1150 // Mux(valid && (pd.isBr) && ans.hit.asBool, 1151 // isWrong ^ (!taken), 1152 // false.B) 1153 // } 1154 // } 1155 1156 // def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1157 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1158 // case (((valid, pd), ans), taken) => 1159 // Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool, 1160 // isWrong ^ (ans.target === commitEntry.target), 1161 // false.B) 1162 // } 1163 // } 1164 1165 // val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B) 1166 // val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B) 1167 // // btb and ubtb pred jal and jalr as well 1168 // val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B) 1169 // val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B) 1170 // val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B) 1171 // val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B) 1172 1173 // val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B) 1174 // val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B) 1175 1176 // val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B) 1177 // val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B) 1178 1179 val perfEvents = Seq( 1180 ("bpu_s2_redirect ", bpu_s2_redirect ), 1181 // ("bpu_s3_redirect ", bpu_s3_redirect ), 1182 ("bpu_to_ftq_stall ", enq.valid && ~enq.ready ), 1183 ("mispredictRedirect ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level), 1184 ("replayRedirect ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level) ), 1185 ("predecodeRedirect ", fromIfuRedirect.valid ), 1186 ("to_ifu_bubble ", io.toIfu.req.ready && !io.toIfu.req.valid ), 1187 ("from_bpu_real_bubble ", !enq.valid && enq.ready && allowBpuIn ), 1188 ("BpInstr ", PopCount(mbpInstrs) ), 1189 ("BpBInstr ", PopCount(mbpBRights | mbpBWrongs) ), 1190 ("BpRight ", PopCount(mbpRights) ), 1191 ("BpWrong ", PopCount(mbpWrongs) ), 1192 ("BpBRight ", PopCount(mbpBRights) ), 1193 ("BpBWrong ", PopCount(mbpBWrongs) ), 1194 ("BpJRight ", PopCount(mbpJRights) ), 1195 ("BpJWrong ", PopCount(mbpJWrongs) ), 1196 ("BpIRight ", PopCount(mbpIRights) ), 1197 ("BpIWrong ", PopCount(mbpIWrongs) ), 1198 ("BpCRight ", PopCount(mbpCRights) ), 1199 ("BpCWrong ", PopCount(mbpCWrongs) ), 1200 ("BpRRight ", PopCount(mbpRRights) ), 1201 ("BpRWrong ", PopCount(mbpRWrongs) ), 1202 ("ftb_false_hit ", PopCount(ftb_false_hit) ), 1203 ("ftb_hit ", PopCount(ftb_hit) ), 1204 ) 1205 generatePerfEvent() 1206} 1207