1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.frontend 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.frontend.icache._ 26import xiangshan.backend.CtrlToFtqIO 27import xiangshan.backend.decode.ImmUnion 28import utility.ChiselDB 29 30class FtqDebugBundle extends Bundle { 31 val pc = UInt(39.W) 32 val target = UInt(39.W) 33 val isBr = Bool() 34 val isJmp = Bool() 35 val isCall = Bool() 36 val isRet = Bool() 37 val misPred = Bool() 38 val isTaken = Bool() 39 val predStage = UInt(2.W) 40} 41 42class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr]( 43 p => p(XSCoreParamsKey).FtqSize 44){ 45} 46 47object FtqPtr { 48 def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = { 49 val ptr = Wire(new FtqPtr) 50 ptr.flag := f 51 ptr.value := v 52 ptr 53 } 54 def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = { 55 apply(!ptr.flag, ptr.value) 56 } 57} 58 59class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule { 60 61 val io = IO(new Bundle() { 62 val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W))) 63 val ren = Input(Vec(numRead, Bool())) 64 val rdata = Output(Vec(numRead, gen)) 65 val waddr = Input(UInt(log2Up(FtqSize).W)) 66 val wen = Input(Bool()) 67 val wdata = Input(gen) 68 }) 69 70 for(i <- 0 until numRead){ 71 val sram = Module(new SRAMTemplate(gen, FtqSize)) 72 sram.io.r.req.valid := io.ren(i) 73 sram.io.r.req.bits.setIdx := io.raddr(i) 74 io.rdata(i) := sram.io.r.resp.data(0) 75 sram.io.w.req.valid := io.wen 76 sram.io.w.req.bits.setIdx := io.waddr 77 sram.io.w.req.bits.data := VecInit(io.wdata) 78 } 79 80} 81 82class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils { 83 val startAddr = UInt(VAddrBits.W) 84 val nextLineAddr = UInt(VAddrBits.W) 85 val isNextMask = Vec(PredictWidth, Bool()) 86 val fallThruError = Bool() 87 // val carry = Bool() 88 def getPc(offset: UInt) = { 89 def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1) 90 def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits) 91 Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)), 92 getOffset(startAddr)+offset, 0.U(instOffsetBits.W)) 93 } 94 def fromBranchPrediction(resp: BranchPredictionBundle) = { 95 def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1) 96 this.startAddr := resp.pc 97 this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U // may be broken on other configs 98 this.isNextMask := VecInit((0 until PredictWidth).map(i => 99 (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool() 100 )) 101 this.fallThruError := resp.fallThruError 102 this 103 } 104 override def toPrintable: Printable = { 105 p"startAddr:${Hexadecimal(startAddr)}" 106 } 107} 108 109class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle { 110 val brMask = Vec(PredictWidth, Bool()) 111 val jmpInfo = ValidUndirectioned(Vec(3, Bool())) 112 val jmpOffset = UInt(log2Ceil(PredictWidth).W) 113 val jalTarget = UInt(VAddrBits.W) 114 val rvcMask = Vec(PredictWidth, Bool()) 115 def hasJal = jmpInfo.valid && !jmpInfo.bits(0) 116 def hasJalr = jmpInfo.valid && jmpInfo.bits(0) 117 def hasCall = jmpInfo.valid && jmpInfo.bits(1) 118 def hasRet = jmpInfo.valid && jmpInfo.bits(2) 119 120 def fromPdWb(pdWb: PredecodeWritebackBundle) = { 121 val pds = pdWb.pd 122 this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid)) 123 this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR 124 this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid), 125 pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet))) 126 this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)) 127 this.rvcMask := VecInit(pds.map(pd => pd.isRVC)) 128 this.jalTarget := pdWb.jalTarget 129 } 130 131 def toPd(offset: UInt) = { 132 require(offset.getWidth == log2Ceil(PredictWidth)) 133 val pd = Wire(new PreDecodeInfo) 134 pd.valid := true.B 135 pd.isRVC := rvcMask(offset) 136 val isBr = brMask(offset) 137 val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0) 138 pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr) 139 pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1) 140 pd.isRet := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2) 141 pd 142 } 143} 144 145 146 147class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends SpeculativeInfo {} 148 149class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst { 150 val meta = UInt(MaxMetaLength.W) 151} 152 153class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle { 154 val target = UInt(VAddrBits.W) 155 val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)) 156} 157 158 159class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle { 160 val ptr = Output(new FtqPtr) 161 val offset = Output(UInt(log2Ceil(PredictWidth).W)) 162 val data = Input(gen) 163 def apply(ptr: FtqPtr, offset: UInt) = { 164 this.ptr := ptr 165 this.offset := offset 166 this.data 167 } 168} 169 170 171class FtqToBpuIO(implicit p: Parameters) extends XSBundle { 172 val redirect = Valid(new BranchPredictionRedirect) 173 val update = Valid(new BranchPredictionUpdate) 174 val enq_ptr = Output(new FtqPtr) 175} 176 177class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper { 178 val req = Decoupled(new FetchRequestBundle) 179 val redirect = Valid(new Redirect) 180 val flushFromBpu = new Bundle { 181 // when ifu pipeline is not stalled, 182 // a packet from bpu s3 can reach f1 at most 183 val s2 = Valid(new FtqPtr) 184 val s3 = Valid(new FtqPtr) 185 def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = { 186 src.valid && !isAfter(src.bits, idx_to_flush) 187 } 188 def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx) 189 def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx) 190 } 191} 192 193class FtqToICacheIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper { 194 //NOTE: req.bits must be prepare in T cycle 195 // while req.valid is set true in T + 1 cycle 196 val req = Decoupled(new FtqToICacheRequestBundle) 197} 198 199trait HasBackendRedirectInfo extends HasXSParameter { 200 def numRedirectPcRead = exuParameters.JmpCnt + exuParameters.AluCnt + 1 201 def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself() 202} 203 204class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo { 205 // write to backend pc mem 206 val pc_mem_wen = Output(Bool()) 207 val pc_mem_waddr = Output(UInt(log2Ceil(FtqSize).W)) 208 val pc_mem_wdata = Output(new Ftq_RF_Components) 209 // newest target 210 val newest_entry_target = Output(UInt(VAddrBits.W)) 211 val newest_entry_ptr = Output(new FtqPtr) 212} 213 214 215class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter { 216 val io = IO(new Bundle { 217 val start_addr = Input(UInt(VAddrBits.W)) 218 val old_entry = Input(new FTBEntry) 219 val pd = Input(new Ftq_pd_Entry) 220 val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W))) 221 val target = Input(UInt(VAddrBits.W)) 222 val hit = Input(Bool()) 223 val mispredict_vec = Input(Vec(PredictWidth, Bool())) 224 225 val new_entry = Output(new FTBEntry) 226 val new_br_insert_pos = Output(Vec(numBr, Bool())) 227 val taken_mask = Output(Vec(numBr, Bool())) 228 val jmp_taken = Output(Bool()) 229 val mispred_mask = Output(Vec(numBr+1, Bool())) 230 231 // for perf counters 232 val is_init_entry = Output(Bool()) 233 val is_old_entry = Output(Bool()) 234 val is_new_br = Output(Bool()) 235 val is_jalr_target_modified = Output(Bool()) 236 val is_always_taken_modified = Output(Bool()) 237 val is_br_full = Output(Bool()) 238 }) 239 240 // no mispredictions detected at predecode 241 val hit = io.hit 242 val pd = io.pd 243 244 val init_entry = WireInit(0.U.asTypeOf(new FTBEntry)) 245 246 247 val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid 248 val entry_has_jmp = pd.jmpInfo.valid 249 val new_jmp_is_jal = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid 250 val new_jmp_is_jalr = entry_has_jmp && pd.jmpInfo.bits(0) && io.cfiIndex.valid 251 val new_jmp_is_call = entry_has_jmp && pd.jmpInfo.bits(1) && io.cfiIndex.valid 252 val new_jmp_is_ret = entry_has_jmp && pd.jmpInfo.bits(2) && io.cfiIndex.valid 253 val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last 254 // val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last 255 256 val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal 257 val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr 258 259 def carryPos = log2Ceil(PredictWidth)+instOffsetBits 260 def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits) 261 // if not hit, establish a new entry 262 init_entry.valid := true.B 263 // tag is left for ftb to assign 264 265 // case br 266 val init_br_slot = init_entry.getSlotForBr(0) 267 when (cfi_is_br) { 268 init_br_slot.valid := true.B 269 init_br_slot.offset := io.cfiIndex.bits 270 init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1) 271 init_entry.always_taken(0) := true.B // set to always taken on init 272 } 273 274 // case jmp 275 when (entry_has_jmp) { 276 init_entry.tailSlot.offset := pd.jmpOffset 277 init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr 278 init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false) 279 } 280 281 val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U) 282 init_entry.pftAddr := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft, getLower(io.start_addr)) 283 init_entry.carry := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft(carryPos-instOffsetBits), true.B) 284 init_entry.isJalr := new_jmp_is_jalr 285 init_entry.isCall := new_jmp_is_call 286 init_entry.isRet := new_jmp_is_ret 287 // that means fall thru points to the middle of an inst 288 init_entry.last_may_be_rvi_call := pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask(pd.jmpOffset) 289 290 // if hit, check whether a new cfi(only br is possible) is detected 291 val oe = io.old_entry 292 val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits) 293 val br_recorded = br_recorded_vec.asUInt.orR 294 val is_new_br = cfi_is_br && !br_recorded 295 val new_br_offset = io.cfiIndex.bits 296 // vec(i) means new br will be inserted BEFORE old br(i) 297 val allBrSlotsVec = oe.allSlotsForBr 298 val new_br_insert_onehot = VecInit((0 until numBr).map{ 299 i => i match { 300 case 0 => 301 !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset 302 case idx => 303 allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset && 304 (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset) 305 } 306 }) 307 308 val old_entry_modified = WireInit(io.old_entry) 309 for (i <- 0 until numBr) { 310 val slot = old_entry_modified.allSlotsForBr(i) 311 when (new_br_insert_onehot(i)) { 312 slot.valid := true.B 313 slot.offset := new_br_offset 314 slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1) 315 old_entry_modified.always_taken(i) := true.B 316 }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) { 317 old_entry_modified.always_taken(i) := false.B 318 // all other fields remain unchanged 319 }.otherwise { 320 // case i == 0, remain unchanged 321 if (i != 0) { 322 val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid 323 when (!noNeedToMoveFromFormerSlot) { 324 slot.fromAnotherSlot(oe.allSlotsForBr(i-1)) 325 old_entry_modified.always_taken(i) := oe.always_taken(i) 326 } 327 } 328 } 329 } 330 331 // two circumstances: 332 // 1. oe: | br | j |, new br should be in front of j, thus addr of j should be new pft 333 // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either 334 // the previous last br or the new br 335 val may_have_to_replace = oe.noEmptySlotForNewBr 336 val pft_need_to_change = is_new_br && may_have_to_replace 337 // it should either be the given last br or the new br 338 when (pft_need_to_change) { 339 val new_pft_offset = 340 Mux(!new_br_insert_onehot.asUInt.orR, 341 new_br_offset, oe.allSlotsForBr.last.offset) 342 343 // set jmp to invalid 344 old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset 345 old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool 346 old_entry_modified.last_may_be_rvi_call := false.B 347 old_entry_modified.isCall := false.B 348 old_entry_modified.isRet := false.B 349 old_entry_modified.isJalr := false.B 350 } 351 352 val old_entry_jmp_target_modified = WireInit(oe) 353 val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits 354 val old_tail_is_jmp = !oe.tailSlot.sharing 355 val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target 356 when (jalr_target_modified) { 357 old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target) 358 old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool())) 359 } 360 361 val old_entry_always_taken = WireInit(oe) 362 val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not 363 for (i <- 0 until numBr) { 364 old_entry_always_taken.always_taken(i) := 365 oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i) 366 always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i) 367 } 368 val always_taken_modified = always_taken_modified_vec.reduce(_||_) 369 370 371 372 val derived_from_old_entry = 373 Mux(is_new_br, old_entry_modified, 374 Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken)) 375 376 377 io.new_entry := Mux(!hit, init_entry, derived_from_old_entry) 378 379 io.new_br_insert_pos := new_br_insert_onehot 380 io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{ 381 case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v 382 }) 383 io.jmp_taken := io.new_entry.jmpValid && io.new_entry.tailSlot.offset === io.cfiIndex.bits 384 for (i <- 0 until numBr) { 385 io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i)) 386 } 387 io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset) 388 389 // for perf counters 390 io.is_init_entry := !hit 391 io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified 392 io.is_new_br := hit && is_new_br 393 io.is_jalr_target_modified := hit && jalr_target_modified 394 io.is_always_taken_modified := hit && always_taken_modified 395 io.is_br_full := hit && is_new_br && may_have_to_replace 396} 397 398class FtqPcMemWrapper(numOtherReads: Int)(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo { 399 val io = IO(new Bundle { 400 val ifuPtr_w = Input(new FtqPtr) 401 val ifuPtrPlus1_w = Input(new FtqPtr) 402 val ifuPtrPlus2_w = Input(new FtqPtr) 403 val commPtr_w = Input(new FtqPtr) 404 val commPtrPlus1_w = Input(new FtqPtr) 405 val ifuPtr_rdata = Output(new Ftq_RF_Components) 406 val ifuPtrPlus1_rdata = Output(new Ftq_RF_Components) 407 val ifuPtrPlus2_rdata = Output(new Ftq_RF_Components) 408 val commPtr_rdata = Output(new Ftq_RF_Components) 409 val commPtrPlus1_rdata = Output(new Ftq_RF_Components) 410 411 val other_raddrs = Input(Vec(numOtherReads, UInt(log2Ceil(FtqSize).W))) 412 val other_rdatas = Output(Vec(numOtherReads, new Ftq_RF_Components)) 413 414 val wen = Input(Bool()) 415 val waddr = Input(UInt(log2Ceil(FtqSize).W)) 416 val wdata = Input(new Ftq_RF_Components) 417 }) 418 419 val num_pc_read = numOtherReads + 5 420 val mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize, 421 num_pc_read, 1, "FtqPC")) 422 mem.io.wen(0) := io.wen 423 mem.io.waddr(0) := io.waddr 424 mem.io.wdata(0) := io.wdata 425 426 // read one cycle ahead for ftq local reads 427 val raddr_vec = VecInit(io.other_raddrs ++ 428 Seq(io.ifuPtr_w.value, io.ifuPtrPlus1_w.value, io.ifuPtrPlus2_w.value, io.commPtrPlus1_w.value, io.commPtr_w.value)) 429 430 mem.io.raddr := raddr_vec 431 432 io.other_rdatas := mem.io.rdata.dropRight(5) 433 io.ifuPtr_rdata := mem.io.rdata.dropRight(4).last 434 io.ifuPtrPlus1_rdata := mem.io.rdata.dropRight(3).last 435 io.ifuPtrPlus2_rdata := mem.io.rdata.dropRight(2).last 436 io.commPtrPlus1_rdata := mem.io.rdata.dropRight(1).last 437 io.commPtr_rdata := mem.io.rdata.last 438} 439 440class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper 441 with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents 442 with HasICacheParameters{ 443 val io = IO(new Bundle { 444 val fromBpu = Flipped(new BpuToFtqIO) 445 val fromIfu = Flipped(new IfuToFtqIO) 446 val fromBackend = Flipped(new CtrlToFtqIO) 447 448 val toBpu = new FtqToBpuIO 449 val toIfu = new FtqToIfuIO 450 val toICache = new FtqToICacheIO 451 val toBackend = new FtqToCtrlIO 452 453 val toPrefetch = new FtqPrefechBundle 454 455 val bpuInfo = new Bundle { 456 val bpRight = Output(UInt(XLEN.W)) 457 val bpWrong = Output(UInt(XLEN.W)) 458 } 459 460 val mmioCommitRead = Flipped(new mmioCommitRead) 461 }) 462 io.bpuInfo := DontCare 463 464 val backendRedirect = Wire(Valid(new Redirect)) 465 val backendRedirectReg = RegNext(backendRedirect) 466 467 val stage2Flush = backendRedirect.valid 468 val backendFlush = stage2Flush || RegNext(stage2Flush) 469 val ifuFlush = Wire(Bool()) 470 471 val flush = stage2Flush || RegNext(stage2Flush) 472 473 val allowBpuIn, allowToIfu = WireInit(false.B) 474 val flushToIfu = !allowToIfu 475 allowBpuIn := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid 476 allowToIfu := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid 477 478 def copyNum = 5 479 val bpuPtr, ifuPtr, ifuWbPtr, commPtr = RegInit(FtqPtr(false.B, 0.U)) 480 val ifuPtrPlus1 = RegInit(FtqPtr(false.B, 1.U)) 481 val ifuPtrPlus2 = RegInit(FtqPtr(false.B, 2.U)) 482 val commPtrPlus1 = RegInit(FtqPtr(false.B, 1.U)) 483 val copied_ifu_ptr = Seq.fill(copyNum)(RegInit(FtqPtr(false.B, 0.U))) 484 val copied_bpu_ptr = Seq.fill(copyNum)(RegInit(FtqPtr(false.B, 0.U))) 485 require(FtqSize >= 4) 486 val ifuPtr_write = WireInit(ifuPtr) 487 val ifuPtrPlus1_write = WireInit(ifuPtrPlus1) 488 val ifuPtrPlus2_write = WireInit(ifuPtrPlus2) 489 val ifuWbPtr_write = WireInit(ifuWbPtr) 490 val commPtr_write = WireInit(commPtr) 491 val commPtrPlus1_write = WireInit(commPtrPlus1) 492 ifuPtr := ifuPtr_write 493 ifuPtrPlus1 := ifuPtrPlus1_write 494 ifuPtrPlus2 := ifuPtrPlus2_write 495 ifuWbPtr := ifuWbPtr_write 496 commPtr := commPtr_write 497 commPtrPlus1 := commPtrPlus1_write 498 copied_ifu_ptr.map{ptr => 499 ptr := ifuPtr_write 500 dontTouch(ptr) 501 } 502 val validEntries = distanceBetween(bpuPtr, commPtr) 503 504 // ********************************************************************** 505 // **************************** enq from bpu **************************** 506 // ********************************************************************** 507 val new_entry_ready = validEntries < FtqSize.U 508 io.fromBpu.resp.ready := new_entry_ready 509 510 val bpu_s2_resp = io.fromBpu.resp.bits.s2 511 val bpu_s3_resp = io.fromBpu.resp.bits.s3 512 val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect 513 val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect 514 515 io.toBpu.enq_ptr := bpuPtr 516 val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1 517 val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect || bpu_s3_redirect) && allowBpuIn 518 519 val bpu_in_resp = io.fromBpu.resp.bits.selectedResp 520 val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx 521 val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx) 522 val bpu_in_resp_idx = bpu_in_resp_ptr.value 523 524 // read ports: prefetchReq ++ ifuReq1 + ifuReq2 + ifuReq3 + commitUpdate2 + commitUpdate 525 val ftq_pc_mem = Module(new FtqPcMemWrapper(1)) 526 // resp from uBTB 527 ftq_pc_mem.io.wen := bpu_in_fire 528 ftq_pc_mem.io.waddr := bpu_in_resp_idx 529 ftq_pc_mem.io.wdata.fromBranchPrediction(bpu_in_resp) 530 531 // ifuRedirect + backendRedirect + commit 532 val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1)) 533 // these info is intended to enq at the last stage of bpu 534 ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid 535 ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value 536 ftq_redirect_sram.io.wdata := io.fromBpu.resp.bits.last_stage_spec_info 537 println(f"ftq redirect SRAM: entry ${ftq_redirect_sram.io.wdata.getWidth} * ${FtqSize} * 3") 538 println(f"ftq redirect SRAM: ahead fh ${ftq_redirect_sram.io.wdata.afhob.getWidth} * ${FtqSize} * 3") 539 540 val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1)) 541 // these info is intended to enq at the last stage of bpu 542 ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid 543 ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value 544 ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.last_stage_meta 545 // ifuRedirect + backendRedirect + commit 546 val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1)) 547 ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid 548 ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value 549 ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.last_stage_ftb_entry 550 551 552 // multi-write 553 val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough //TODO: remove this 554 val newest_entry_target = Reg(UInt(VAddrBits.W)) 555 val newest_entry_ptr = Reg(new FtqPtr) 556 val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W)))) 557 val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool()))) 558 val pred_stage = Reg(Vec(FtqSize, UInt(2.W))) 559 560 val c_invalid :: c_valid :: c_commited :: Nil = Enum(3) 561 val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) { 562 VecInit(Seq.fill(PredictWidth)(c_invalid)) 563 })) 564 565 val f_to_send :: f_sent :: Nil = Enum(2) 566 val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent))) 567 568 val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3) 569 val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit))) 570 571 // modify registers one cycle later to cut critical path 572 val last_cycle_bpu_in = RegNext(bpu_in_fire) 573 val last_cycle_bpu_in_ptr = RegNext(bpu_in_resp_ptr) 574 val last_cycle_bpu_in_idx = last_cycle_bpu_in_ptr.value 575 val last_cycle_bpu_target = RegNext(bpu_in_resp.getTarget) 576 val last_cycle_cfiIndex = RegNext(bpu_in_resp.cfiIndex) 577 val last_cycle_bpu_in_stage = RegNext(bpu_in_stage) 578 579 def extra_copyNum_for_commitStateQueue = 2 580 val copied_last_cycle_bpu_in = VecInit(Seq.fill(copyNum+extra_copyNum_for_commitStateQueue)(RegNext(bpu_in_fire))) 581 val copied_last_cycle_bpu_in_ptr_for_ftq = VecInit(Seq.fill(extra_copyNum_for_commitStateQueue)(RegNext(bpu_in_resp_ptr))) 582 583 when (last_cycle_bpu_in) { 584 entry_fetch_status(last_cycle_bpu_in_idx) := f_to_send 585 cfiIndex_vec(last_cycle_bpu_in_idx) := last_cycle_cfiIndex 586 pred_stage(last_cycle_bpu_in_idx) := last_cycle_bpu_in_stage 587 588 update_target(last_cycle_bpu_in_idx) := last_cycle_bpu_target // TODO: remove this 589 newest_entry_target := last_cycle_bpu_target 590 newest_entry_ptr := last_cycle_bpu_in_ptr 591 } 592 593 // reduce fanout by delay write for a cycle 594 when (RegNext(last_cycle_bpu_in)) { 595 mispredict_vec(RegNext(last_cycle_bpu_in_idx)) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B))) 596 } 597 598 // reduce fanout using copied last_cycle_bpu_in and copied last_cycle_bpu_in_ptr 599 val copied_last_cycle_bpu_in_for_ftq = copied_last_cycle_bpu_in.takeRight(extra_copyNum_for_commitStateQueue) 600 copied_last_cycle_bpu_in_for_ftq.zip(copied_last_cycle_bpu_in_ptr_for_ftq).zipWithIndex.map { 601 case ((in, ptr), i) => 602 when (in) { 603 val perSetEntries = FtqSize / extra_copyNum_for_commitStateQueue // 32 604 require(FtqSize % extra_copyNum_for_commitStateQueue == 0) 605 for (j <- 0 until perSetEntries) { 606 when (ptr.value === (i*perSetEntries+j).U) { 607 commitStateQueue(i*perSetEntries+j) := VecInit(Seq.fill(PredictWidth)(c_invalid)) 608 } 609 } 610 } 611 } 612 613 // num cycle is fixed 614 io.toBackend.newest_entry_ptr := RegNext(newest_entry_ptr) 615 io.toBackend.newest_entry_target := RegNext(newest_entry_target) 616 617 618 bpuPtr := bpuPtr + enq_fire 619 copied_bpu_ptr.map(_ := bpuPtr + enq_fire) 620 when (io.toIfu.req.fire && allowToIfu) { 621 ifuPtr_write := ifuPtrPlus1 622 ifuPtrPlus1_write := ifuPtrPlus2 623 ifuPtrPlus2_write := ifuPtrPlus2 + 1.U 624 } 625 626 // only use ftb result to assign hit status 627 when (bpu_s2_resp.valid) { 628 entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit) 629 } 630 631 632 io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect 633 io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx 634 when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) { 635 bpuPtr := bpu_s2_resp.ftq_idx + 1.U 636 copied_bpu_ptr.map(_ := bpu_s2_resp.ftq_idx + 1.U) 637 // only when ifuPtr runs ahead of bpu s2 resp should we recover it 638 when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) { 639 ifuPtr_write := bpu_s2_resp.ftq_idx 640 ifuPtrPlus1_write := bpu_s2_resp.ftq_idx + 1.U 641 ifuPtrPlus2_write := bpu_s2_resp.ftq_idx + 2.U 642 } 643 } 644 645 io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect 646 io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx 647 when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) { 648 bpuPtr := bpu_s3_resp.ftq_idx + 1.U 649 copied_bpu_ptr.map(_ := bpu_s3_resp.ftq_idx + 1.U) 650 // only when ifuPtr runs ahead of bpu s2 resp should we recover it 651 when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) { 652 ifuPtr_write := bpu_s3_resp.ftq_idx 653 ifuPtrPlus1_write := bpu_s3_resp.ftq_idx + 1.U 654 ifuPtrPlus2_write := bpu_s3_resp.ftq_idx + 2.U 655 } 656 } 657 658 XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n") 659 660 (0 until copyNum).map{i => 661 XSError(copied_bpu_ptr(i) =/= bpuPtr, "\ncopiedBpuPtr is different from bpuPtr!\n") 662 } 663 664 // **************************************************************** 665 // **************************** to ifu **************************** 666 // **************************************************************** 667 // 0 for ifu, and 1-4 for ICache 668 val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata, enable=bpu_in_fire) 669 val copied_bpu_in_bypass_buf = VecInit(Seq.fill(copyNum)(RegEnable(ftq_pc_mem.io.wdata, enable=bpu_in_fire))) 670 val bpu_in_bypass_buf_for_ifu = bpu_in_bypass_buf 671 val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr) 672 val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire) 673 674 val copied_bpu_in_bypass_ptr = VecInit(Seq.fill(copyNum)(RegNext(bpu_in_resp_ptr))) 675 val copied_last_cycle_to_ifu_fire = VecInit(Seq.fill(copyNum)(RegNext(io.toIfu.req.fire))) 676 677 // read pc and target 678 ftq_pc_mem.io.ifuPtr_w := ifuPtr_write 679 ftq_pc_mem.io.ifuPtrPlus1_w := ifuPtrPlus1_write 680 ftq_pc_mem.io.ifuPtrPlus2_w := ifuPtrPlus2_write 681 ftq_pc_mem.io.commPtr_w := commPtr_write 682 ftq_pc_mem.io.commPtrPlus1_w := commPtrPlus1_write 683 684 685 io.toIfu.req.bits.ftqIdx := ifuPtr 686 687 val toICachePcBundle = Wire(Vec(copyNum,new Ftq_RF_Components)) 688 val toICacheEntryToSend = Wire(Vec(copyNum,Bool())) 689 val toIfuPcBundle = Wire(new Ftq_RF_Components) 690 val entry_is_to_send = WireInit(entry_fetch_status(ifuPtr.value) === f_to_send) 691 val entry_ftq_offset = WireInit(cfiIndex_vec(ifuPtr.value)) 692 val entry_next_addr = Wire(UInt(VAddrBits.W)) 693 694 val pc_mem_ifu_ptr_rdata = VecInit(Seq.fill(copyNum)(RegNext(ftq_pc_mem.io.ifuPtr_rdata))) 695 val pc_mem_ifu_plus1_rdata = VecInit(Seq.fill(copyNum)(RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata))) 696 val diff_entry_next_addr = WireInit(update_target(ifuPtr.value)) //TODO: remove this 697 698 val copied_ifu_plus1_to_send = VecInit(Seq.fill(copyNum)(RegNext(entry_fetch_status(ifuPtrPlus1.value) === f_to_send) || RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1)))) 699 val copied_ifu_ptr_to_send = VecInit(Seq.fill(copyNum)(RegNext(entry_fetch_status(ifuPtr.value) === f_to_send) || RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr))) 700 701 for(i <- 0 until copyNum){ 702 when(copied_last_cycle_bpu_in(i) && copied_bpu_in_bypass_ptr(i) === copied_ifu_ptr(i)){ 703 toICachePcBundle(i) := copied_bpu_in_bypass_buf(i) 704 toICacheEntryToSend(i) := true.B 705 }.elsewhen(copied_last_cycle_to_ifu_fire(i)){ 706 toICachePcBundle(i) := pc_mem_ifu_plus1_rdata(i) 707 toICacheEntryToSend(i) := copied_ifu_plus1_to_send(i) 708 }.otherwise{ 709 toICachePcBundle(i) := pc_mem_ifu_ptr_rdata(i) 710 toICacheEntryToSend(i) := copied_ifu_ptr_to_send(i) 711 } 712 } 713 714 // TODO: reconsider target address bypass logic 715 when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) { 716 toIfuPcBundle := bpu_in_bypass_buf_for_ifu 717 entry_is_to_send := true.B 718 entry_next_addr := last_cycle_bpu_target 719 entry_ftq_offset := last_cycle_cfiIndex 720 diff_entry_next_addr := last_cycle_bpu_target // TODO: remove this 721 }.elsewhen (last_cycle_to_ifu_fire) { 722 toIfuPcBundle := RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata) 723 entry_is_to_send := RegNext(entry_fetch_status(ifuPtrPlus1.value) === f_to_send) || 724 RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1)) // reduce potential bubbles 725 entry_next_addr := Mux(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1), 726 bpu_in_bypass_buf_for_ifu.startAddr, 727 Mux(ifuPtr === newest_entry_ptr, 728 newest_entry_target, 729 RegNext(ftq_pc_mem.io.ifuPtrPlus2_rdata.startAddr))) // ifuPtr+2 730 }.otherwise { 731 toIfuPcBundle := RegNext(ftq_pc_mem.io.ifuPtr_rdata) 732 entry_is_to_send := RegNext(entry_fetch_status(ifuPtr.value) === f_to_send) || 733 RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) // reduce potential bubbles 734 entry_next_addr := Mux(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1), 735 bpu_in_bypass_buf_for_ifu.startAddr, 736 Mux(ifuPtr === newest_entry_ptr, 737 newest_entry_target, 738 RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata.startAddr))) // ifuPtr+1 739 } 740 741 io.toIfu.req.valid := entry_is_to_send && ifuPtr =/= bpuPtr 742 io.toIfu.req.bits.nextStartAddr := entry_next_addr 743 io.toIfu.req.bits.ftqOffset := entry_ftq_offset 744 io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle) 745 746 io.toICache.req.valid := entry_is_to_send && ifuPtr =/= bpuPtr 747 io.toICache.req.bits.readValid.zipWithIndex.map{case(copy, i) => copy := toICacheEntryToSend(i) && copied_ifu_ptr(i) =/= copied_bpu_ptr(i)} 748 io.toICache.req.bits.pcMemRead.zipWithIndex.map{case(copy,i) => copy.fromFtqPcBundle(toICachePcBundle(i))} 749 // io.toICache.req.bits.bypassSelect := last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr 750 // io.toICache.req.bits.bpuBypassWrite.zipWithIndex.map{case(bypassWrtie, i) => 751 // bypassWrtie.startAddr := bpu_in_bypass_buf.tail(i).startAddr 752 // bypassWrtie.nextlineStart := bpu_in_bypass_buf.tail(i).nextLineAddr 753 // } 754 755 // TODO: remove this 756 XSError(io.toIfu.req.valid && diff_entry_next_addr =/= entry_next_addr, 757 p"\nifu_req_target wrong! ifuPtr: ${ifuPtr}, entry_next_addr: ${Hexadecimal(entry_next_addr)} diff_entry_next_addr: ${Hexadecimal(diff_entry_next_addr)}\n") 758 759 // when fall through is smaller in value than start address, there must be a false hit 760 when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) { 761 when (io.toIfu.req.fire && 762 !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) && 763 !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr) 764 ) { 765 entry_hit_status(ifuPtr.value) := h_false_hit 766 // XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr) 767 } 768 XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr) 769 } 770 771 XSPerfAccumulate(f"fall_through_error_to_ifu", toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit && 772 io.toIfu.req.fire && !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) && !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr)) 773 774 val ifu_req_should_be_flushed = 775 io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx) || 776 io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx) 777 778 when (io.toIfu.req.fire && !ifu_req_should_be_flushed) { 779 entry_fetch_status(ifuPtr.value) := f_sent 780 } 781 782 // ********************************************************************* 783 // **************************** wb from ifu **************************** 784 // ********************************************************************* 785 val pdWb = io.fromIfu.pdWb 786 val pds = pdWb.bits.pd 787 val ifu_wb_valid = pdWb.valid 788 val ifu_wb_idx = pdWb.bits.ftqIdx.value 789 // read ports: commit update 790 val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1)) 791 ftq_pd_mem.io.wen(0) := ifu_wb_valid 792 ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value 793 ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits) 794 795 val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid 796 val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid 797 val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B) 798 val pd_reg = RegEnable(pds, pdWb.valid) 799 val start_pc_reg = RegEnable(pdWb.bits.pc(0), pdWb.valid) 800 val wb_idx_reg = RegEnable(ifu_wb_idx, pdWb.valid) 801 802 when (ifu_wb_valid) { 803 val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{ 804 case (v, inRange) => v && inRange 805 }) 806 (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{ 807 case (qe, v) => when (v) { qe := c_valid } 808 } 809 } 810 811 when (ifu_wb_valid) { 812 ifuWbPtr_write := ifuWbPtr + 1.U 813 } 814 815 ftb_entry_mem.io.raddr.head := ifu_wb_idx 816 val has_false_hit = WireInit(false.B) 817 when (RegNext(hit_pd_valid)) { 818 // check for false hit 819 val pred_ftb_entry = ftb_entry_mem.io.rdata.head 820 val brSlots = pred_ftb_entry.brSlots 821 val tailSlot = pred_ftb_entry.tailSlot 822 // we check cfis that bpu predicted 823 824 // bpu predicted branches but denied by predecode 825 val br_false_hit = 826 brSlots.map{ 827 s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr) 828 }.reduce(_||_) || 829 (tailSlot.valid && pred_ftb_entry.tailSlot.sharing && 830 !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr)) 831 832 val jmpOffset = tailSlot.offset 833 val jmp_pd = pd_reg(jmpOffset) 834 val jal_false_hit = pred_ftb_entry.jmpValid && 835 ((pred_ftb_entry.isJal && !(jmp_pd.valid && jmp_pd.isJal)) || 836 (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) || 837 (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) || 838 (pred_ftb_entry.isRet && !(jmp_pd.valid && jmp_pd.isRet)) 839 ) 840 841 has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg 842 XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0)) 843 844 // assert(!has_false_hit) 845 } 846 847 when (has_false_hit) { 848 entry_hit_status(wb_idx_reg) := h_false_hit 849 } 850 851 852 // ********************************************************************** 853 // ***************************** to backend ***************************** 854 // ********************************************************************** 855 // to backend pc mem / target 856 io.toBackend.pc_mem_wen := RegNext(last_cycle_bpu_in) 857 io.toBackend.pc_mem_waddr := RegNext(last_cycle_bpu_in_idx) 858 io.toBackend.pc_mem_wdata := RegNext(bpu_in_bypass_buf_for_ifu) 859 860 // ******************************************************************************* 861 // **************************** redirect from backend **************************** 862 // ******************************************************************************* 863 864 // redirect read cfiInfo, couples to redirectGen s2 865 ftq_redirect_sram.io.ren.init.last := backendRedirect.valid 866 ftq_redirect_sram.io.raddr.init.last := backendRedirect.bits.ftqIdx.value 867 868 ftb_entry_mem.io.raddr.init.last := backendRedirect.bits.ftqIdx.value 869 870 val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last 871 val fromBackendRedirect = WireInit(backendRedirectReg) 872 val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate 873 backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo) 874 875 val r_ftb_entry = ftb_entry_mem.io.rdata.init.last 876 val r_ftqOffset = fromBackendRedirect.bits.ftqOffset 877 878 when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) { 879 backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +& 880 (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) && 881 !r_ftb_entry.newBrCanNotInsert(r_ftqOffset)) 882 883 backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) || 884 !r_ftb_entry.newBrCanNotInsert(r_ftqOffset)) 885 }.otherwise { 886 backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt 887 backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt 888 } 889 890 891 // *************************************************************************** 892 // **************************** redirect from ifu **************************** 893 // *************************************************************************** 894 val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect))) 895 fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush 896 fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx 897 fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits 898 fromIfuRedirect.bits.level := RedirectLevel.flushAfter 899 900 val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate 901 ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits) 902 ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits) 903 ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid 904 ifuRedirectCfiUpdate.target := pdWb.bits.target 905 ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid 906 ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid 907 908 val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect))) 909 val ifuRedirectToBpu = WireInit(ifuRedirectReg) 910 ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid 911 912 ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid 913 ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value 914 915 ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value 916 917 val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate 918 toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head) 919 when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) { 920 toBpuCfi.target := toBpuCfi.rasEntry.retAddr 921 } 922 923 // ********************************************************************* 924 // **************************** wb from exu **************************** 925 // ********************************************************************* 926 927 backendRedirect := io.fromBackend.redirect 928 929 def extractRedirectInfo(wb: Valid[Redirect]) = { 930 val ftqPtr = wb.bits.ftqIdx 931 val ftqOffset = wb.bits.ftqOffset 932 val taken = wb.bits.cfiUpdate.taken 933 val mispred = wb.bits.cfiUpdate.isMisPred 934 (wb.valid, ftqPtr, ftqOffset, taken, mispred) 935 } 936 937 // fix mispredict entry 938 val lastIsMispredict = RegNext( 939 backendRedirect.valid && backendRedirect.bits.level === RedirectLevel.flushAfter, init = false.B 940 ) 941 942 def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = { 943 val (r_valid, r_ptr, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect) 944 val r_idx = r_ptr.value 945 val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits 946 val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits 947 when (cfiIndex_bits_wen || cfiIndex_valid_wen) { 948 cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken 949 } .elsewhen (r_valid && !r_taken && r_offset =/= cfiIndex_vec(r_idx).bits) { 950 cfiIndex_vec(r_idx).valid :=false.B 951 } 952 when (cfiIndex_bits_wen) { 953 cfiIndex_vec(r_idx).bits := r_offset 954 } 955 newest_entry_target := redirect.bits.cfiUpdate.target 956 newest_entry_ptr := r_ptr 957 update_target(r_idx) := redirect.bits.cfiUpdate.target // TODO: remove this 958 if (isBackend) { 959 mispredict_vec(r_idx)(r_offset) := r_mispred 960 } 961 } 962 963 when(backendRedirectReg.valid) { 964 updateCfiInfo(backendRedirectReg) 965 }.elsewhen (ifuRedirectToBpu.valid) { 966 updateCfiInfo(ifuRedirectToBpu, isBackend=false) 967 } 968 969 // *********************************************************************************** 970 // **************************** flush ptr and state queue **************************** 971 // *********************************************************************************** 972 973 val redirectVec = VecInit(backendRedirect, fromIfuRedirect) 974 975 // when redirect, we should reset ptrs and status queues 976 when(redirectVec.map(r => r.valid).reduce(_||_)){ 977 val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits))) 978 val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_) 979 val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level)) 980 val next = idx + 1.U 981 bpuPtr := next 982 copied_bpu_ptr.map(_ := next) 983 ifuPtr_write := next 984 ifuWbPtr_write := next 985 ifuPtrPlus1_write := idx + 2.U 986 ifuPtrPlus2_write := idx + 3.U 987 988 } 989 when(RegNext(redirectVec.map(r => r.valid).reduce(_||_))){ 990 val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits))) 991 val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_) 992 val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level)) 993 when (RegNext(notIfu)) { 994 commitStateQueue(RegNext(idx.value)).zipWithIndex.foreach({ case (s, i) => 995 when(i.U > RegNext(offset) || i.U === RegNext(offset) && RegNext(flushItSelf)){ 996 s := c_invalid 997 } 998 }) 999 } 1000 } 1001 1002 1003 // only the valid bit is actually needed 1004 io.toIfu.redirect.bits := backendRedirect.bits 1005 io.toIfu.redirect.valid := stage2Flush 1006 1007 // commit 1008 for (c <- io.fromBackend.rob_commits) { 1009 when(c.valid) { 1010 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited 1011 // TODO: remove this 1012 // For instruction fusions, we also update the next instruction 1013 when (c.bits.commitType === 4.U) { 1014 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited 1015 }.elsewhen(c.bits.commitType === 5.U) { 1016 commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited 1017 }.elsewhen(c.bits.commitType === 6.U) { 1018 val index = (c.bits.ftqIdx + 1.U).value 1019 commitStateQueue(index)(0) := c_commited 1020 }.elsewhen(c.bits.commitType === 7.U) { 1021 val index = (c.bits.ftqIdx + 1.U).value 1022 commitStateQueue(index)(1) := c_commited 1023 } 1024 } 1025 } 1026 1027 // **************************************************************** 1028 // **************************** to bpu **************************** 1029 // **************************************************************** 1030 1031 io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu) 1032 1033 val may_have_stall_from_bpu = Wire(Bool()) 1034 val bpu_ftb_update_stall = RegInit(0.U(2.W)) // 2-cycle stall, so we need 3 states 1035 may_have_stall_from_bpu := bpu_ftb_update_stall =/= 0.U 1036 val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu && 1037 Cat(commitStateQueue(commPtr.value).map(s => { 1038 s === c_invalid || s === c_commited 1039 })).andR() 1040 1041 val mmioReadPtr = io.mmioCommitRead.mmioFtqPtr 1042 val mmioLastCommit = isBefore(commPtr, mmioReadPtr) && (isAfter(ifuPtr,mmioReadPtr) || mmioReadPtr === ifuPtr) && 1043 Cat(commitStateQueue(mmioReadPtr.value).map(s => { s === c_invalid || s === c_commited})).andR() 1044 io.mmioCommitRead.mmioLastCommit := RegNext(mmioLastCommit) 1045 1046 // commit reads 1047 val commit_pc_bundle = RegNext(ftq_pc_mem.io.commPtr_rdata) 1048 val commit_target = 1049 Mux(RegNext(commPtr === newest_entry_ptr), 1050 RegNext(newest_entry_target), 1051 RegNext(ftq_pc_mem.io.commPtrPlus1_rdata.startAddr)) 1052 ftq_pd_mem.io.raddr.last := commPtr.value 1053 val commit_pd = ftq_pd_mem.io.rdata.last 1054 ftq_redirect_sram.io.ren.last := canCommit 1055 ftq_redirect_sram.io.raddr.last := commPtr.value 1056 val commit_spec_meta = ftq_redirect_sram.io.rdata.last 1057 ftq_meta_1r_sram.io.ren(0) := canCommit 1058 ftq_meta_1r_sram.io.raddr(0) := commPtr.value 1059 val commit_meta = ftq_meta_1r_sram.io.rdata(0) 1060 ftb_entry_mem.io.raddr.last := commPtr.value 1061 val commit_ftb_entry = ftb_entry_mem.io.rdata.last 1062 1063 // need one cycle to read mem and srams 1064 val do_commit_ptr = RegNext(commPtr) 1065 val do_commit = RegNext(canCommit, init=false.B) 1066 when (canCommit) { 1067 commPtr_write := commPtrPlus1 1068 commPtrPlus1_write := commPtrPlus1 + 1.U 1069 } 1070 val commit_state = RegNext(commitStateQueue(commPtr.value)) 1071 val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value)) 1072 // 1073 //when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) { 1074 // can_commit_cfi.valid := false.B 1075 //} 1076 val commit_cfi = RegNext(can_commit_cfi) 1077 val debug_cfi = RegNext(commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited && can_commit_cfi.valid) 1078 1079 val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map { 1080 case (mis, state) => mis && state === c_commited 1081 }) 1082 val can_commit_hit = entry_hit_status(commPtr.value) 1083 val commit_hit = RegNext(can_commit_hit) 1084 val diff_commit_target = RegNext(update_target(commPtr.value)) // TODO: remove this 1085 val commit_stage = RegNext(pred_stage(commPtr.value)) 1086 val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken 1087 1088 val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit 1089 switch (bpu_ftb_update_stall) { 1090 is (0.U) { 1091 when (can_commit_cfi.valid && !to_bpu_hit && canCommit) { 1092 bpu_ftb_update_stall := 2.U // 2-cycle stall 1093 } 1094 } 1095 is (2.U) { 1096 bpu_ftb_update_stall := 1.U 1097 } 1098 is (1.U) { 1099 bpu_ftb_update_stall := 0.U 1100 } 1101 is (3.U) { 1102 XSError(true.B, "bpu_ftb_update_stall should be 0, 1 or 2") 1103 } 1104 } 1105 1106 // TODO: remove this 1107 XSError(do_commit && diff_commit_target =/= commit_target, "\ncommit target should be the same as update target\n") 1108 1109 io.toBpu.update := DontCare 1110 io.toBpu.update.valid := commit_valid && do_commit 1111 val update = io.toBpu.update.bits 1112 update.false_hit := commit_hit === h_false_hit 1113 update.pc := commit_pc_bundle.startAddr 1114 update.meta := commit_meta.meta 1115 update.cfi_idx := commit_cfi 1116 update.full_target := commit_target 1117 update.from_stage := commit_stage 1118 update.spec_info := commit_spec_meta 1119 XSError(commit_valid && do_commit && debug_cfi, "\ncommit cfi can be non c_commited\n") 1120 1121 val commit_real_hit = commit_hit === h_hit 1122 val update_ftb_entry = update.ftb_entry 1123 1124 val ftbEntryGen = Module(new FTBEntryGen).io 1125 ftbEntryGen.start_addr := commit_pc_bundle.startAddr 1126 ftbEntryGen.old_entry := commit_ftb_entry 1127 ftbEntryGen.pd := commit_pd 1128 ftbEntryGen.cfiIndex := commit_cfi 1129 ftbEntryGen.target := commit_target 1130 ftbEntryGen.hit := commit_real_hit 1131 ftbEntryGen.mispredict_vec := commit_mispredict 1132 1133 update_ftb_entry := ftbEntryGen.new_entry 1134 update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos 1135 update.mispred_mask := ftbEntryGen.mispred_mask 1136 update.old_entry := ftbEntryGen.is_old_entry 1137 update.pred_hit := commit_hit === h_hit || commit_hit === h_false_hit 1138 update.br_taken_mask := ftbEntryGen.taken_mask 1139 update.jmp_taken := ftbEntryGen.jmp_taken 1140 1141 // update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc) 1142 // update.full_pred.jalr_target := commit_target 1143 // update.full_pred.hit := true.B 1144 // when (update.full_pred.is_jalr) { 1145 // update.full_pred.targets.last := commit_target 1146 // } 1147 1148 // **************************************************************** 1149 // *********************** to prefetch **************************** 1150 // **************************************************************** 1151 1152 ftq_pc_mem.io.other_raddrs(0) := DontCare 1153 if(cacheParams.hasPrefetch){ 1154 val prefetchPtr = RegInit(FtqPtr(false.B, 0.U)) 1155 val diff_prefetch_addr = WireInit(update_target(prefetchPtr.value)) //TODO: remove this 1156 1157 prefetchPtr := prefetchPtr + io.toPrefetch.req.fire() 1158 1159 ftq_pc_mem.io.other_raddrs(0) := prefetchPtr.value 1160 1161 when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s2_resp.ftq_idx)) { 1162 prefetchPtr := bpu_s2_resp.ftq_idx 1163 } 1164 1165 when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s3_resp.ftq_idx)) { 1166 prefetchPtr := bpu_s3_resp.ftq_idx 1167 // XSError(true.B, "\ns3_redirect mechanism not implemented!\n") 1168 } 1169 1170 1171 val prefetch_is_to_send = WireInit(entry_fetch_status(prefetchPtr.value) === f_to_send) 1172 val prefetch_addr = Wire(UInt(VAddrBits.W)) 1173 1174 when (last_cycle_bpu_in && bpu_in_bypass_ptr === prefetchPtr) { 1175 prefetch_is_to_send := true.B 1176 prefetch_addr := last_cycle_bpu_target 1177 diff_prefetch_addr := last_cycle_bpu_target // TODO: remove this 1178 }.otherwise{ 1179 prefetch_addr := RegNext( ftq_pc_mem.io.other_rdatas(0).startAddr) 1180 } 1181 io.toPrefetch.req.valid := prefetchPtr =/= bpuPtr && prefetch_is_to_send 1182 io.toPrefetch.req.bits.target := prefetch_addr 1183 1184 when(redirectVec.map(r => r.valid).reduce(_||_)){ 1185 val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits))) 1186 val next = r.ftqIdx + 1.U 1187 prefetchPtr := next 1188 } 1189 1190 // TODO: remove this 1191 // XSError(io.toPrefetch.req.valid && diff_prefetch_addr =/= prefetch_addr, 1192 // f"\nprefetch_req_target wrong! prefetchPtr: ${prefetchPtr}, prefetch_addr: ${Hexadecimal(prefetch_addr)} diff_prefetch_addr: ${Hexadecimal(diff_prefetch_addr)}\n") 1193 1194 1195 XSError(isBefore(bpuPtr, prefetchPtr) && !isFull(bpuPtr, prefetchPtr), "\nprefetchPtr is before bpuPtr!\n") 1196 XSError(isBefore(prefetchPtr, ifuPtr) && !isFull(ifuPtr, prefetchPtr), "\nifuPtr is before prefetchPtr!\n") 1197 } 1198 else { 1199 io.toPrefetch.req <> DontCare 1200 } 1201 1202 // ****************************************************************************** 1203 // **************************** commit perf counters **************************** 1204 // ****************************************************************************** 1205 1206 val commit_inst_mask = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt 1207 val commit_mispred_mask = commit_mispredict.asUInt 1208 val commit_not_mispred_mask = ~commit_mispred_mask 1209 1210 val commit_br_mask = commit_pd.brMask.asUInt 1211 val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W))) 1212 val commit_cfi_mask = (commit_br_mask | commit_jmp_mask) 1213 1214 val mbpInstrs = commit_inst_mask & commit_cfi_mask 1215 1216 val mbpRights = mbpInstrs & commit_not_mispred_mask 1217 val mbpWrongs = mbpInstrs & commit_mispred_mask 1218 1219 io.bpuInfo.bpRight := PopCount(mbpRights) 1220 io.bpuInfo.bpWrong := PopCount(mbpWrongs) 1221 1222 val ftqBranchTraceDB = ChiselDB.createTable("FTQTable" + p(XSCoreParamsKey).HartId.toString, new FtqDebugBundle) 1223 // Cfi Info 1224 for (i <- 0 until PredictWidth) { 1225 val pc = commit_pc_bundle.startAddr + (i * instBytes).U 1226 val v = commit_state(i) === c_commited 1227 val isBr = commit_pd.brMask(i) 1228 val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U 1229 val isCfi = isBr || isJmp 1230 val isTaken = commit_cfi.valid && commit_cfi.bits === i.U 1231 val misPred = commit_mispredict(i) 1232 // val ghist = commit_spec_meta.ghist.predHist 1233 val histPtr = commit_spec_meta.histPtr 1234 val predCycle = commit_meta.meta(63, 0) 1235 val target = commit_target 1236 1237 val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}))) 1238 val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_) 1239 val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid)) 1240 XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " + 1241 p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " + 1242 p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " + 1243 p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n") 1244 1245 val logbundle = Wire(new FtqDebugBundle) 1246 logbundle.pc := pc 1247 logbundle.target := target 1248 logbundle.isBr := isBr 1249 logbundle.isJmp := isJmp 1250 logbundle.isCall := isJmp && commit_pd.hasCall 1251 logbundle.isRet := isJmp && commit_pd.hasRet 1252 logbundle.misPred := misPred 1253 logbundle.isTaken := isTaken 1254 logbundle.predStage := commit_stage 1255 1256 ftqBranchTraceDB.log( 1257 data = logbundle /* hardware of type T */, 1258 en = v && do_commit && isCfi, 1259 site = "FTQ" + p(XSCoreParamsKey).HartId.toString, 1260 clock = clock, 1261 reset = reset 1262 ) 1263 } 1264 1265 val enq = io.fromBpu.resp 1266 val perf_redirect = backendRedirect 1267 1268 XSPerfAccumulate("entry", validEntries) 1269 XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready) 1270 XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level) 1271 XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)) 1272 XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid) 1273 1274 XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid) 1275 1276 XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready) 1277 XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn) 1278 XSPerfAccumulate("bpu_to_ifu_bubble", bpuPtr === ifuPtr) 1279 1280 val from_bpu = io.fromBpu.resp.bits 1281 def in_entry_len_map_gen(resp: BpuToFtqBundle)(stage: String) = { 1282 val entry_len = (resp.last_stage_ftb_entry.getFallThrough(resp.s3.pc) - resp.s3.pc) >> instOffsetBits 1283 val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U) 1284 val entry_len_map = (1 to PredictWidth+1).map(i => 1285 f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.s3.valid) 1286 ).foldLeft(Map[String, UInt]())(_+_) 1287 entry_len_map 1288 } 1289 val s3_entry_len_map = in_entry_len_map_gen(from_bpu)("s3") 1290 1291 val to_ifu = io.toIfu.req.bits 1292 1293 1294 1295 val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U) 1296 val commit_num_inst_map = (1 to PredictWidth).map(i => 1297 f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit) 1298 ).foldLeft(Map[String, UInt]())(_+_) 1299 1300 1301 1302 val commit_jal_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W))) 1303 val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W))) 1304 val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W))) 1305 val commit_ret_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W))) 1306 1307 1308 val mbpBRights = mbpRights & commit_br_mask 1309 val mbpJRights = mbpRights & commit_jal_mask 1310 val mbpIRights = mbpRights & commit_jalr_mask 1311 val mbpCRights = mbpRights & commit_call_mask 1312 val mbpRRights = mbpRights & commit_ret_mask 1313 1314 val mbpBWrongs = mbpWrongs & commit_br_mask 1315 val mbpJWrongs = mbpWrongs & commit_jal_mask 1316 val mbpIWrongs = mbpWrongs & commit_jalr_mask 1317 val mbpCWrongs = mbpWrongs & commit_call_mask 1318 val mbpRWrongs = mbpWrongs & commit_ret_mask 1319 1320 val commit_pred_stage = RegNext(pred_stage(commPtr.value)) 1321 1322 def pred_stage_map(src: UInt, name: String) = { 1323 (0 until numBpStages).map(i => 1324 f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i))) 1325 ).foldLeft(Map[String, UInt]())(_+_) 1326 } 1327 1328 val mispred_stage_map = pred_stage_map(mbpWrongs, "mispredict") 1329 val br_mispred_stage_map = pred_stage_map(mbpBWrongs, "br_mispredict") 1330 val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict") 1331 val correct_stage_map = pred_stage_map(mbpRights, "correct") 1332 val br_correct_stage_map = pred_stage_map(mbpBRights, "br_correct") 1333 val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct") 1334 1335 val update_valid = io.toBpu.update.valid 1336 def u(cond: Bool) = update_valid && cond 1337 val ftb_false_hit = u(update.false_hit) 1338 // assert(!ftb_false_hit) 1339 val ftb_hit = u(commit_hit === h_hit) 1340 1341 val ftb_new_entry = u(ftbEntryGen.is_init_entry) 1342 val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid 1343 val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0) 1344 val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid 1345 1346 val ftb_old_entry = u(ftbEntryGen.is_old_entry) 1347 1348 val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified) 1349 val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br) 1350 val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified) 1351 val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full 1352 val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified 1353 1354 val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits 1355 val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U) 1356 val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i => 1357 f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry) 1358 ).foldLeft(Map[String, UInt]())(_+_) 1359 val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i => 1360 f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry) 1361 ).foldLeft(Map[String, UInt]())(_+_) 1362 1363 val ftq_occupancy_map = (0 to FtqSize).map(i => 1364 f"ftq_has_entry_$i" ->( validEntries === i.U) 1365 ).foldLeft(Map[String, UInt]())(_+_) 1366 1367 val perfCountsMap = Map( 1368 "BpInstr" -> PopCount(mbpInstrs), 1369 "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs), 1370 "BpRight" -> PopCount(mbpRights), 1371 "BpWrong" -> PopCount(mbpWrongs), 1372 "BpBRight" -> PopCount(mbpBRights), 1373 "BpBWrong" -> PopCount(mbpBWrongs), 1374 "BpJRight" -> PopCount(mbpJRights), 1375 "BpJWrong" -> PopCount(mbpJWrongs), 1376 "BpIRight" -> PopCount(mbpIRights), 1377 "BpIWrong" -> PopCount(mbpIWrongs), 1378 "BpCRight" -> PopCount(mbpCRights), 1379 "BpCWrong" -> PopCount(mbpCWrongs), 1380 "BpRRight" -> PopCount(mbpRRights), 1381 "BpRWrong" -> PopCount(mbpRWrongs), 1382 1383 "ftb_false_hit" -> PopCount(ftb_false_hit), 1384 "ftb_hit" -> PopCount(ftb_hit), 1385 "ftb_new_entry" -> PopCount(ftb_new_entry), 1386 "ftb_new_entry_only_br" -> PopCount(ftb_new_entry_only_br), 1387 "ftb_new_entry_only_jmp" -> PopCount(ftb_new_entry_only_jmp), 1388 "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp), 1389 "ftb_old_entry" -> PopCount(ftb_old_entry), 1390 "ftb_modified_entry" -> PopCount(ftb_modified_entry), 1391 "ftb_modified_entry_new_br" -> PopCount(ftb_modified_entry_new_br), 1392 "ftb_jalr_target_modified" -> PopCount(ftb_modified_entry_jalr_target_modified), 1393 "ftb_modified_entry_br_full" -> PopCount(ftb_modified_entry_br_full), 1394 "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken) 1395 ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++ 1396 s3_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++ 1397 mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++ 1398 correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map 1399 1400 for((key, value) <- perfCountsMap) { 1401 XSPerfAccumulate(key, value) 1402 } 1403 1404 // --------------------------- Debug -------------------------------- 1405 // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable) 1406 XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable) 1407 XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n") 1408 XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n") 1409 XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " + 1410 p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n") 1411 XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n") 1412 1413 // def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1414 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1415 // case (((valid, pd), ans), taken) => 1416 // Mux(valid && pd.isBr, 1417 // isWrong ^ Mux(ans.hit.asBool, 1418 // Mux(ans.taken.asBool, taken && ans.target === commitEntry.target, 1419 // !taken), 1420 // !taken), 1421 // false.B) 1422 // } 1423 // } 1424 1425 // def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1426 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1427 // case (((valid, pd), ans), taken) => 1428 // Mux(valid && pd.isBr, 1429 // isWrong ^ Mux(ans.hit.asBool, 1430 // Mux(ans.taken.asBool, taken && ans.target === commitEntry.target, 1431 // !taken), 1432 // !taken), 1433 // false.B) 1434 // } 1435 // } 1436 1437 // def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1438 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1439 // case (((valid, pd), ans), taken) => 1440 // Mux(valid && pd.isBr, 1441 // isWrong ^ (ans.taken.asBool === taken), 1442 // false.B) 1443 // } 1444 // } 1445 1446 // def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1447 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1448 // case (((valid, pd), ans), taken) => 1449 // Mux(valid && (pd.isBr) && ans.hit.asBool, 1450 // isWrong ^ (!taken), 1451 // false.B) 1452 // } 1453 // } 1454 1455 // def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = { 1456 // commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map { 1457 // case (((valid, pd), ans), taken) => 1458 // Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool, 1459 // isWrong ^ (ans.target === commitEntry.target), 1460 // false.B) 1461 // } 1462 // } 1463 1464 // val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B) 1465 // val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B) 1466 // // btb and ubtb pred jal and jalr as well 1467 // val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B) 1468 // val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B) 1469 // val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B) 1470 // val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B) 1471 1472 // val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B) 1473 // val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B) 1474 1475 // val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B) 1476 // val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B) 1477 1478 val perfEvents = Seq( 1479 ("bpu_s2_redirect ", bpu_s2_redirect ), 1480 ("bpu_s3_redirect ", bpu_s3_redirect ), 1481 ("bpu_to_ftq_stall ", enq.valid && ~enq.ready ), 1482 ("mispredictRedirect ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level), 1483 ("replayRedirect ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level) ), 1484 ("predecodeRedirect ", fromIfuRedirect.valid ), 1485 ("to_ifu_bubble ", io.toIfu.req.ready && !io.toIfu.req.valid ), 1486 ("from_bpu_real_bubble ", !enq.valid && enq.ready && allowBpuIn ), 1487 ("BpInstr ", PopCount(mbpInstrs) ), 1488 ("BpBInstr ", PopCount(mbpBRights | mbpBWrongs) ), 1489 ("BpRight ", PopCount(mbpRights) ), 1490 ("BpWrong ", PopCount(mbpWrongs) ), 1491 ("BpBRight ", PopCount(mbpBRights) ), 1492 ("BpBWrong ", PopCount(mbpBWrongs) ), 1493 ("BpJRight ", PopCount(mbpJRights) ), 1494 ("BpJWrong ", PopCount(mbpJWrongs) ), 1495 ("BpIRight ", PopCount(mbpIRights) ), 1496 ("BpIWrong ", PopCount(mbpIWrongs) ), 1497 ("BpCRight ", PopCount(mbpCRights) ), 1498 ("BpCWrong ", PopCount(mbpCWrongs) ), 1499 ("BpRRight ", PopCount(mbpRRights) ), 1500 ("BpRWrong ", PopCount(mbpRWrongs) ), 1501 ("ftb_false_hit ", PopCount(ftb_false_hit) ), 1502 ("ftb_hit ", PopCount(ftb_hit) ), 1503 ) 1504 generatePerfEvent() 1505}