xref: /XiangShan/src/main/scala/xiangshan/frontend/NewFtq.scala (revision 7be982afbfec69ab7df73f9086c708ffd2d07422)
1/***************************************************************************************
2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences
3* Copyright (c) 2020-2021 Peng Cheng Laboratory
4*
5* XiangShan is licensed under Mulan PSL v2.
6* You can use this software according to the terms and conditions of the Mulan PSL v2.
7* You may obtain a copy of Mulan PSL v2 at:
8*          http://license.coscl.org.cn/MulanPSL2
9*
10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
13*
14* See the Mulan PSL v2 for more details.
15***************************************************************************************/
16
17package xiangshan.frontend
18
19import chipsalliance.rocketchip.config.Parameters
20import chisel3._
21import chisel3.util._
22import utils._
23import xiangshan._
24import xiangshan.frontend.icache._
25import xiangshan.backend.CtrlToFtqIO
26import xiangshan.backend.decode.ImmUnion
27
28class FtqPtr(implicit p: Parameters) extends CircularQueuePtr[FtqPtr](
29  p => p(XSCoreParamsKey).FtqSize
30){
31}
32
33object FtqPtr {
34  def apply(f: Bool, v: UInt)(implicit p: Parameters): FtqPtr = {
35    val ptr = Wire(new FtqPtr)
36    ptr.flag := f
37    ptr.value := v
38    ptr
39  }
40  def inverse(ptr: FtqPtr)(implicit p: Parameters): FtqPtr = {
41    apply(!ptr.flag, ptr.value)
42  }
43}
44
45class FtqNRSRAM[T <: Data](gen: T, numRead: Int)(implicit p: Parameters) extends XSModule {
46
47  val io = IO(new Bundle() {
48    val raddr = Input(Vec(numRead, UInt(log2Up(FtqSize).W)))
49    val ren = Input(Vec(numRead, Bool()))
50    val rdata = Output(Vec(numRead, gen))
51    val waddr = Input(UInt(log2Up(FtqSize).W))
52    val wen = Input(Bool())
53    val wdata = Input(gen)
54  })
55
56  for(i <- 0 until numRead){
57    val sram = Module(new SRAMTemplate(gen, FtqSize))
58    sram.io.r.req.valid := io.ren(i)
59    sram.io.r.req.bits.setIdx := io.raddr(i)
60    io.rdata(i) := sram.io.r.resp.data(0)
61    sram.io.w.req.valid := io.wen
62    sram.io.w.req.bits.setIdx := io.waddr
63    sram.io.w.req.bits.data := VecInit(io.wdata)
64  }
65
66}
67
68class Ftq_RF_Components(implicit p: Parameters) extends XSBundle with BPUUtils {
69  val startAddr = UInt(VAddrBits.W)
70  val nextLineAddr = UInt(VAddrBits.W)
71  val isNextMask = Vec(PredictWidth, Bool())
72  val fallThruError = Bool()
73  // val carry = Bool()
74  def getPc(offset: UInt) = {
75    def getHigher(pc: UInt) = pc(VAddrBits-1, log2Ceil(PredictWidth)+instOffsetBits+1)
76    def getOffset(pc: UInt) = pc(log2Ceil(PredictWidth)+instOffsetBits, instOffsetBits)
77    Cat(getHigher(Mux(isNextMask(offset) && startAddr(log2Ceil(PredictWidth)+instOffsetBits), nextLineAddr, startAddr)),
78        getOffset(startAddr)+offset, 0.U(instOffsetBits.W))
79  }
80  def fromBranchPrediction(resp: BranchPredictionBundle) = {
81    def carryPos(addr: UInt) = addr(instOffsetBits+log2Ceil(PredictWidth)+1)
82    this.startAddr := resp.pc
83    this.nextLineAddr := resp.pc + (FetchWidth * 4 * 2).U // may be broken on other configs
84    this.isNextMask := VecInit((0 until PredictWidth).map(i =>
85      (resp.pc(log2Ceil(PredictWidth), 1) +& i.U)(log2Ceil(PredictWidth)).asBool()
86    ))
87    this.fallThruError := resp.fallThruError
88    this
89  }
90  override def toPrintable: Printable = {
91    p"startAddr:${Hexadecimal(startAddr)}"
92  }
93}
94
95class Ftq_pd_Entry(implicit p: Parameters) extends XSBundle {
96  val brMask = Vec(PredictWidth, Bool())
97  val jmpInfo = ValidUndirectioned(Vec(3, Bool()))
98  val jmpOffset = UInt(log2Ceil(PredictWidth).W)
99  val jalTarget = UInt(VAddrBits.W)
100  val rvcMask = Vec(PredictWidth, Bool())
101  def hasJal  = jmpInfo.valid && !jmpInfo.bits(0)
102  def hasJalr = jmpInfo.valid && jmpInfo.bits(0)
103  def hasCall = jmpInfo.valid && jmpInfo.bits(1)
104  def hasRet  = jmpInfo.valid && jmpInfo.bits(2)
105
106  def fromPdWb(pdWb: PredecodeWritebackBundle) = {
107    val pds = pdWb.pd
108    this.brMask := VecInit(pds.map(pd => pd.isBr && pd.valid))
109    this.jmpInfo.valid := VecInit(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid)).asUInt.orR
110    this.jmpInfo.bits := ParallelPriorityMux(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid),
111                                             pds.map(pd => VecInit(pd.isJalr, pd.isCall, pd.isRet)))
112    this.jmpOffset := ParallelPriorityEncoder(pds.map(pd => (pd.isJal || pd.isJalr) && pd.valid))
113    this.rvcMask := VecInit(pds.map(pd => pd.isRVC))
114    this.jalTarget := pdWb.jalTarget
115  }
116
117  def toPd(offset: UInt) = {
118    require(offset.getWidth == log2Ceil(PredictWidth))
119    val pd = Wire(new PreDecodeInfo)
120    pd.valid := true.B
121    pd.isRVC := rvcMask(offset)
122    val isBr = brMask(offset)
123    val isJalr = offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(0)
124    pd.brType := Cat(offset === jmpOffset && jmpInfo.valid, isJalr || isBr)
125    pd.isCall := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(1)
126    pd.isRet  := offset === jmpOffset && jmpInfo.valid && jmpInfo.bits(2)
127    pd
128  }
129}
130
131
132
133class Ftq_Redirect_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
134  val rasSp = UInt(log2Ceil(RasSize).W)
135  val rasEntry = new RASEntry
136  // val specCnt = Vec(numBr, UInt(10.W))
137  // val ghist = new ShiftingGlobalHistory
138  val folded_hist = new AllFoldedHistories(foldedGHistInfos)
139  val afhob = new AllAheadFoldedHistoryOldestBits(foldedGHistInfos)
140  val lastBrNumOH = UInt((numBr+1).W)
141
142  val histPtr = new CGHPtr
143
144  def fromBranchPrediction(resp: BranchPredictionBundle) = {
145    assert(!resp.is_minimal)
146    this.rasSp := resp.rasSp
147    this.rasEntry := resp.rasTop
148    this.folded_hist := resp.folded_hist
149    this.afhob := resp.afhob
150    this.lastBrNumOH := resp.lastBrNumOH
151    this.histPtr := resp.histPtr
152    this
153  }
154}
155
156class Ftq_1R_SRAMEntry(implicit p: Parameters) extends XSBundle with HasBPUConst {
157  val meta = UInt(MaxMetaLength.W)
158}
159
160class Ftq_Pred_Info(implicit p: Parameters) extends XSBundle {
161  val target = UInt(VAddrBits.W)
162  val cfiIndex = ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))
163}
164
165
166class FtqRead[T <: Data](private val gen: T)(implicit p: Parameters) extends XSBundle {
167  val ptr = Output(new FtqPtr)
168  val offset = Output(UInt(log2Ceil(PredictWidth).W))
169  val data = Input(gen)
170  def apply(ptr: FtqPtr, offset: UInt) = {
171    this.ptr := ptr
172    this.offset := offset
173    this.data
174  }
175}
176
177
178class FtqToBpuIO(implicit p: Parameters) extends XSBundle {
179  val redirect = Valid(new BranchPredictionRedirect)
180  val update = Valid(new BranchPredictionUpdate)
181  val enq_ptr = Output(new FtqPtr)
182}
183
184class FtqToIfuIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper {
185  val req = Decoupled(new FetchRequestBundle)
186  val redirect = Valid(new Redirect)
187  val flushFromBpu = new Bundle {
188    // when ifu pipeline is not stalled,
189    // a packet from bpu s3 can reach f1 at most
190    val s2 = Valid(new FtqPtr)
191    val s3 = Valid(new FtqPtr)
192    def shouldFlushBy(src: Valid[FtqPtr], idx_to_flush: FtqPtr) = {
193      src.valid && !isAfter(src.bits, idx_to_flush)
194    }
195    def shouldFlushByStage2(idx: FtqPtr) = shouldFlushBy(s2, idx)
196    def shouldFlushByStage3(idx: FtqPtr) = shouldFlushBy(s3, idx)
197  }
198}
199
200class FtqToICacheIO(implicit p: Parameters) extends XSBundle with HasCircularQueuePtrHelper {
201  //NOTE: req.bits must be prepare in T cycle
202  // while req.valid is set true in T + 1 cycle
203  val req = Decoupled(new FtqToICacheRequestBundle)
204}
205
206trait HasBackendRedirectInfo extends HasXSParameter {
207  def numRedirectPcRead = exuParameters.JmpCnt + exuParameters.AluCnt + 1
208  def isLoadReplay(r: Valid[Redirect]) = r.bits.flushItself()
209}
210
211class FtqToCtrlIO(implicit p: Parameters) extends XSBundle with HasBackendRedirectInfo {
212  // write to backend pc mem
213  val pc_mem_wen = Output(Bool())
214  val pc_mem_waddr = Output(UInt(log2Ceil(FtqSize).W))
215  val pc_mem_wdata = Output(new Ftq_RF_Components)
216  // newest target
217  val newest_entry_target = Output(UInt(VAddrBits.W))
218  val newest_entry_ptr = Output(new FtqPtr)
219}
220
221
222class FTBEntryGen(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo with HasBPUParameter {
223  val io = IO(new Bundle {
224    val start_addr = Input(UInt(VAddrBits.W))
225    val old_entry = Input(new FTBEntry)
226    val pd = Input(new Ftq_pd_Entry)
227    val cfiIndex = Flipped(Valid(UInt(log2Ceil(PredictWidth).W)))
228    val target = Input(UInt(VAddrBits.W))
229    val hit = Input(Bool())
230    val mispredict_vec = Input(Vec(PredictWidth, Bool()))
231
232    val new_entry = Output(new FTBEntry)
233    val new_br_insert_pos = Output(Vec(numBr, Bool()))
234    val taken_mask = Output(Vec(numBr, Bool()))
235    val mispred_mask = Output(Vec(numBr+1, Bool()))
236
237    // for perf counters
238    val is_init_entry = Output(Bool())
239    val is_old_entry = Output(Bool())
240    val is_new_br = Output(Bool())
241    val is_jalr_target_modified = Output(Bool())
242    val is_always_taken_modified = Output(Bool())
243    val is_br_full = Output(Bool())
244  })
245
246  // no mispredictions detected at predecode
247  val hit = io.hit
248  val pd = io.pd
249
250  val init_entry = WireInit(0.U.asTypeOf(new FTBEntry))
251
252
253  val cfi_is_br = pd.brMask(io.cfiIndex.bits) && io.cfiIndex.valid
254  val entry_has_jmp = pd.jmpInfo.valid
255  val new_jmp_is_jal  = entry_has_jmp && !pd.jmpInfo.bits(0) && io.cfiIndex.valid
256  val new_jmp_is_jalr = entry_has_jmp &&  pd.jmpInfo.bits(0) && io.cfiIndex.valid
257  val new_jmp_is_call = entry_has_jmp &&  pd.jmpInfo.bits(1) && io.cfiIndex.valid
258  val new_jmp_is_ret  = entry_has_jmp &&  pd.jmpInfo.bits(2) && io.cfiIndex.valid
259  val last_jmp_rvi = entry_has_jmp && pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask.last
260  // val last_br_rvi = cfi_is_br && io.cfiIndex.bits === (PredictWidth-1).U && !pd.rvcMask.last
261
262  val cfi_is_jal = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jal
263  val cfi_is_jalr = io.cfiIndex.bits === pd.jmpOffset && new_jmp_is_jalr
264
265  def carryPos = log2Ceil(PredictWidth)+instOffsetBits
266  def getLower(pc: UInt) = pc(carryPos-1, instOffsetBits)
267  // if not hit, establish a new entry
268  init_entry.valid := true.B
269  // tag is left for ftb to assign
270
271  // case br
272  val init_br_slot = init_entry.getSlotForBr(0)
273  when (cfi_is_br) {
274    init_br_slot.valid := true.B
275    init_br_slot.offset := io.cfiIndex.bits
276    init_br_slot.setLowerStatByTarget(io.start_addr, io.target, numBr == 1)
277    init_entry.always_taken(0) := true.B // set to always taken on init
278  }
279
280  // case jmp
281  when (entry_has_jmp) {
282    init_entry.tailSlot.offset := pd.jmpOffset
283    init_entry.tailSlot.valid := new_jmp_is_jal || new_jmp_is_jalr
284    init_entry.tailSlot.setLowerStatByTarget(io.start_addr, Mux(cfi_is_jalr, io.target, pd.jalTarget), isShare=false)
285  }
286
287  val jmpPft = getLower(io.start_addr) +& pd.jmpOffset +& Mux(pd.rvcMask(pd.jmpOffset), 1.U, 2.U)
288  init_entry.pftAddr := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft, getLower(io.start_addr))
289  init_entry.carry   := Mux(entry_has_jmp && !last_jmp_rvi, jmpPft(carryPos-instOffsetBits), true.B)
290  init_entry.isJalr := new_jmp_is_jalr
291  init_entry.isCall := new_jmp_is_call
292  init_entry.isRet  := new_jmp_is_ret
293  // that means fall thru points to the middle of an inst
294  init_entry.last_may_be_rvi_call := pd.jmpOffset === (PredictWidth-1).U && !pd.rvcMask(pd.jmpOffset)
295
296  // if hit, check whether a new cfi(only br is possible) is detected
297  val oe = io.old_entry
298  val br_recorded_vec = oe.getBrRecordedVec(io.cfiIndex.bits)
299  val br_recorded = br_recorded_vec.asUInt.orR
300  val is_new_br = cfi_is_br && !br_recorded
301  val new_br_offset = io.cfiIndex.bits
302  // vec(i) means new br will be inserted BEFORE old br(i)
303  val allBrSlotsVec = oe.allSlotsForBr
304  val new_br_insert_onehot = VecInit((0 until numBr).map{
305    i => i match {
306      case 0 =>
307        !allBrSlotsVec(0).valid || new_br_offset < allBrSlotsVec(0).offset
308      case idx =>
309        allBrSlotsVec(idx-1).valid && new_br_offset > allBrSlotsVec(idx-1).offset &&
310        (!allBrSlotsVec(idx).valid || new_br_offset < allBrSlotsVec(idx).offset)
311    }
312  })
313
314  val old_entry_modified = WireInit(io.old_entry)
315  for (i <- 0 until numBr) {
316    val slot = old_entry_modified.allSlotsForBr(i)
317    when (new_br_insert_onehot(i)) {
318      slot.valid := true.B
319      slot.offset := new_br_offset
320      slot.setLowerStatByTarget(io.start_addr, io.target, i == numBr-1)
321      old_entry_modified.always_taken(i) := true.B
322    }.elsewhen (new_br_offset > oe.allSlotsForBr(i).offset) {
323      old_entry_modified.always_taken(i) := false.B
324      // all other fields remain unchanged
325    }.otherwise {
326      // case i == 0, remain unchanged
327      if (i != 0) {
328        val noNeedToMoveFromFormerSlot = (i == numBr-1).B && !oe.brSlots.last.valid
329        when (!noNeedToMoveFromFormerSlot) {
330          slot.fromAnotherSlot(oe.allSlotsForBr(i-1))
331          old_entry_modified.always_taken(i) := oe.always_taken(i)
332        }
333      }
334    }
335  }
336
337  // two circumstances:
338  // 1. oe: | br | j  |, new br should be in front of j, thus addr of j should be new pft
339  // 2. oe: | br | br |, new br could be anywhere between, thus new pft is the addr of either
340  //        the previous last br or the new br
341  val may_have_to_replace = oe.noEmptySlotForNewBr
342  val pft_need_to_change = is_new_br && may_have_to_replace
343  // it should either be the given last br or the new br
344  when (pft_need_to_change) {
345    val new_pft_offset =
346      Mux(!new_br_insert_onehot.asUInt.orR,
347        new_br_offset, oe.allSlotsForBr.last.offset)
348
349    // set jmp to invalid
350    old_entry_modified.pftAddr := getLower(io.start_addr) + new_pft_offset
351    old_entry_modified.carry := (getLower(io.start_addr) +& new_pft_offset).head(1).asBool
352    old_entry_modified.last_may_be_rvi_call := false.B
353    old_entry_modified.isCall := false.B
354    old_entry_modified.isRet := false.B
355    old_entry_modified.isJalr := false.B
356  }
357
358  val old_entry_jmp_target_modified = WireInit(oe)
359  val old_target = oe.tailSlot.getTarget(io.start_addr) // may be wrong because we store only 20 lowest bits
360  val old_tail_is_jmp = !oe.tailSlot.sharing
361  val jalr_target_modified = cfi_is_jalr && (old_target =/= io.target) && old_tail_is_jmp // TODO: pass full jalr target
362  when (jalr_target_modified) {
363    old_entry_jmp_target_modified.setByJmpTarget(io.start_addr, io.target)
364    old_entry_jmp_target_modified.always_taken := 0.U.asTypeOf(Vec(numBr, Bool()))
365  }
366
367  val old_entry_always_taken = WireInit(oe)
368  val always_taken_modified_vec = Wire(Vec(numBr, Bool())) // whether modified or not
369  for (i <- 0 until numBr) {
370    old_entry_always_taken.always_taken(i) :=
371      oe.always_taken(i) && io.cfiIndex.valid && oe.brValids(i) && io.cfiIndex.bits === oe.brOffset(i)
372    always_taken_modified_vec(i) := oe.always_taken(i) && !old_entry_always_taken.always_taken(i)
373  }
374  val always_taken_modified = always_taken_modified_vec.reduce(_||_)
375
376
377
378  val derived_from_old_entry =
379    Mux(is_new_br, old_entry_modified,
380      Mux(jalr_target_modified, old_entry_jmp_target_modified, old_entry_always_taken))
381
382
383  io.new_entry := Mux(!hit, init_entry, derived_from_old_entry)
384
385  io.new_br_insert_pos := new_br_insert_onehot
386  io.taken_mask := VecInit((io.new_entry.brOffset zip io.new_entry.brValids).map{
387    case (off, v) => io.cfiIndex.bits === off && io.cfiIndex.valid && v
388  })
389  for (i <- 0 until numBr) {
390    io.mispred_mask(i) := io.new_entry.brValids(i) && io.mispredict_vec(io.new_entry.brOffset(i))
391  }
392  io.mispred_mask.last := io.new_entry.jmpValid && io.mispredict_vec(pd.jmpOffset)
393
394  // for perf counters
395  io.is_init_entry := !hit
396  io.is_old_entry := hit && !is_new_br && !jalr_target_modified && !always_taken_modified
397  io.is_new_br := hit && is_new_br
398  io.is_jalr_target_modified := hit && jalr_target_modified
399  io.is_always_taken_modified := hit && always_taken_modified
400  io.is_br_full := hit && is_new_br && may_have_to_replace
401}
402
403class FtqPcMemWrapper(numOtherReads: Int)(implicit p: Parameters) extends XSModule with HasBackendRedirectInfo {
404  val io = IO(new Bundle {
405    val ifuPtr_w       = Input(new FtqPtr)
406    val ifuPtrPlus1_w  = Input(new FtqPtr)
407    val ifuPtrPlus2_w  = Input(new FtqPtr)
408    val commPtr_w      = Input(new FtqPtr)
409    val commPtrPlus1_w = Input(new FtqPtr)
410    val ifuPtr_rdata       = Output(new Ftq_RF_Components)
411    val ifuPtrPlus1_rdata  = Output(new Ftq_RF_Components)
412    val ifuPtrPlus2_rdata  = Output(new Ftq_RF_Components)
413    val commPtr_rdata      = Output(new Ftq_RF_Components)
414    val commPtrPlus1_rdata = Output(new Ftq_RF_Components)
415
416    val other_raddrs = Input(Vec(numOtherReads, UInt(log2Ceil(FtqSize).W)))
417    val other_rdatas = Output(Vec(numOtherReads, new Ftq_RF_Components))
418
419    val wen = Input(Bool())
420    val waddr = Input(UInt(log2Ceil(FtqSize).W))
421    val wdata = Input(new Ftq_RF_Components)
422  })
423
424  val num_pc_read = numOtherReads + 5
425  val mem = Module(new SyncDataModuleTemplate(new Ftq_RF_Components, FtqSize,
426    num_pc_read, 1, "FtqPC"))
427  mem.io.wen(0)   := io.wen
428  mem.io.waddr(0) := io.waddr
429  mem.io.wdata(0) := io.wdata
430
431  // read one cycle ahead for ftq local reads
432  val raddr_vec = VecInit(io.other_raddrs ++
433    Seq(io.ifuPtr_w.value, io.ifuPtrPlus1_w.value, io.ifuPtrPlus2_w.value, io.commPtrPlus1_w.value, io.commPtr_w.value))
434
435  mem.io.raddr := raddr_vec
436
437  io.other_rdatas       := mem.io.rdata.dropRight(5)
438  io.ifuPtr_rdata       := mem.io.rdata.dropRight(4).last
439  io.ifuPtrPlus1_rdata  := mem.io.rdata.dropRight(3).last
440  io.ifuPtrPlus2_rdata  := mem.io.rdata.dropRight(2).last
441  io.commPtrPlus1_rdata := mem.io.rdata.dropRight(1).last
442  io.commPtr_rdata      := mem.io.rdata.last
443}
444
445class Ftq(implicit p: Parameters) extends XSModule with HasCircularQueuePtrHelper
446  with HasBackendRedirectInfo with BPUUtils with HasBPUConst with HasPerfEvents
447  with HasICacheParameters{
448  val io = IO(new Bundle {
449    val fromBpu = Flipped(new BpuToFtqIO)
450    val fromIfu = Flipped(new IfuToFtqIO)
451    val fromBackend = Flipped(new CtrlToFtqIO)
452
453    val toBpu = new FtqToBpuIO
454    val toIfu = new FtqToIfuIO
455    val toICache = new FtqToICacheIO
456    val toBackend = new FtqToCtrlIO
457
458    val toPrefetch = new FtqPrefechBundle
459
460    val bpuInfo = new Bundle {
461      val bpRight = Output(UInt(XLEN.W))
462      val bpWrong = Output(UInt(XLEN.W))
463    }
464  })
465  io.bpuInfo := DontCare
466
467  val backendRedirect = Wire(Valid(new Redirect))
468  val backendRedirectReg = RegNext(backendRedirect)
469
470  val stage2Flush = backendRedirect.valid
471  val backendFlush = stage2Flush || RegNext(stage2Flush)
472  val ifuFlush = Wire(Bool())
473
474  val flush = stage2Flush || RegNext(stage2Flush)
475
476  val allowBpuIn, allowToIfu = WireInit(false.B)
477  val flushToIfu = !allowToIfu
478  allowBpuIn := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid
479  allowToIfu := !ifuFlush && !backendRedirect.valid && !backendRedirectReg.valid
480
481  def copyNum = 5
482  val bpuPtr, ifuPtr, ifuWbPtr, commPtr = RegInit(FtqPtr(false.B, 0.U))
483  val ifuPtrPlus1 = RegInit(FtqPtr(false.B, 1.U))
484  val ifuPtrPlus2 = RegInit(FtqPtr(false.B, 2.U))
485  val commPtrPlus1 = RegInit(FtqPtr(false.B, 1.U))
486  val copied_ifu_ptr = Seq.fill(copyNum)(RegInit(FtqPtr(false.B, 0.U)))
487  val copied_bpu_ptr = Seq.fill(copyNum)(RegInit(FtqPtr(false.B, 0.U)))
488  require(FtqSize >= 4)
489  val ifuPtr_write       = WireInit(ifuPtr)
490  val ifuPtrPlus1_write  = WireInit(ifuPtrPlus1)
491  val ifuPtrPlus2_write  = WireInit(ifuPtrPlus2)
492  val ifuWbPtr_write     = WireInit(ifuWbPtr)
493  val commPtr_write      = WireInit(commPtr)
494  val commPtrPlus1_write = WireInit(commPtrPlus1)
495  ifuPtr       := ifuPtr_write
496  ifuPtrPlus1  := ifuPtrPlus1_write
497  ifuPtrPlus2  := ifuPtrPlus2_write
498  ifuWbPtr     := ifuWbPtr_write
499  commPtr      := commPtr_write
500  commPtrPlus1 := commPtrPlus1_write
501  copied_ifu_ptr.map{ptr =>
502    ptr := ifuPtr_write
503    dontTouch(ptr)
504  }
505  val validEntries = distanceBetween(bpuPtr, commPtr)
506
507  // **********************************************************************
508  // **************************** enq from bpu ****************************
509  // **********************************************************************
510  val new_entry_ready = validEntries < FtqSize.U
511  io.fromBpu.resp.ready := new_entry_ready
512
513  val bpu_s2_resp = io.fromBpu.resp.bits.s2
514  val bpu_s3_resp = io.fromBpu.resp.bits.s3
515  val bpu_s2_redirect = bpu_s2_resp.valid && bpu_s2_resp.hasRedirect
516  val bpu_s3_redirect = bpu_s3_resp.valid && bpu_s3_resp.hasRedirect
517
518  io.toBpu.enq_ptr := bpuPtr
519  val enq_fire = io.fromBpu.resp.fire() && allowBpuIn // from bpu s1
520  val bpu_in_fire = (io.fromBpu.resp.fire() || bpu_s2_redirect || bpu_s3_redirect) && allowBpuIn
521
522  val bpu_in_resp = io.fromBpu.resp.bits.selectedResp
523  val bpu_in_stage = io.fromBpu.resp.bits.selectedRespIdx
524  val bpu_in_resp_ptr = Mux(bpu_in_stage === BP_S1, bpuPtr, bpu_in_resp.ftq_idx)
525  val bpu_in_resp_idx = bpu_in_resp_ptr.value
526
527  // read ports:      prefetchReq ++  ifuReq1 + ifuReq2 + ifuReq3 + commitUpdate2 + commitUpdate
528  val ftq_pc_mem = Module(new FtqPcMemWrapper(1))
529  // resp from uBTB
530  ftq_pc_mem.io.wen := bpu_in_fire
531  ftq_pc_mem.io.waddr := bpu_in_resp_idx
532  ftq_pc_mem.io.wdata.fromBranchPrediction(bpu_in_resp)
533
534  //                                                            ifuRedirect + backendRedirect + commit
535  val ftq_redirect_sram = Module(new FtqNRSRAM(new Ftq_Redirect_SRAMEntry, 1+1+1))
536  // these info is intended to enq at the last stage of bpu
537  ftq_redirect_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
538  ftq_redirect_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
539  ftq_redirect_sram.io.wdata.fromBranchPrediction(io.fromBpu.resp.bits.lastStage)
540  println(f"ftq redirect SRAM: entry ${ftq_redirect_sram.io.wdata.getWidth} * ${FtqSize} * 3")
541  println(f"ftq redirect SRAM: ahead fh ${ftq_redirect_sram.io.wdata.afhob.getWidth} * ${FtqSize} * 3")
542
543  val ftq_meta_1r_sram = Module(new FtqNRSRAM(new Ftq_1R_SRAMEntry, 1))
544  // these info is intended to enq at the last stage of bpu
545  ftq_meta_1r_sram.io.wen := io.fromBpu.resp.bits.lastStage.valid
546  ftq_meta_1r_sram.io.waddr := io.fromBpu.resp.bits.lastStage.ftq_idx.value
547  ftq_meta_1r_sram.io.wdata.meta := io.fromBpu.resp.bits.meta
548  //                                                            ifuRedirect + backendRedirect + commit
549  val ftb_entry_mem = Module(new SyncDataModuleTemplate(new FTBEntry, FtqSize, 1+1+1, 1))
550  ftb_entry_mem.io.wen(0) := io.fromBpu.resp.bits.lastStage.valid
551  ftb_entry_mem.io.waddr(0) := io.fromBpu.resp.bits.lastStage.ftq_idx.value
552  ftb_entry_mem.io.wdata(0) := io.fromBpu.resp.bits.lastStage.ftb_entry
553
554
555  // multi-write
556  val update_target = Reg(Vec(FtqSize, UInt(VAddrBits.W))) // could be taken target or fallThrough //TODO: remove this
557  val newest_entry_target = Reg(UInt(VAddrBits.W))
558  val newest_entry_ptr = Reg(new FtqPtr)
559  val cfiIndex_vec = Reg(Vec(FtqSize, ValidUndirectioned(UInt(log2Ceil(PredictWidth).W))))
560  val mispredict_vec = Reg(Vec(FtqSize, Vec(PredictWidth, Bool())))
561  val pred_stage = Reg(Vec(FtqSize, UInt(2.W)))
562
563  val c_invalid :: c_valid :: c_commited :: Nil = Enum(3)
564  val commitStateQueue = RegInit(VecInit(Seq.fill(FtqSize) {
565    VecInit(Seq.fill(PredictWidth)(c_invalid))
566  }))
567
568  val f_to_send :: f_sent :: Nil = Enum(2)
569  val entry_fetch_status = RegInit(VecInit(Seq.fill(FtqSize)(f_sent)))
570
571  val h_not_hit :: h_false_hit :: h_hit :: Nil = Enum(3)
572  val entry_hit_status = RegInit(VecInit(Seq.fill(FtqSize)(h_not_hit)))
573
574  // modify registers one cycle later to cut critical path
575  val last_cycle_bpu_in = RegNext(bpu_in_fire)
576  val last_cycle_bpu_in_ptr = RegNext(bpu_in_resp_ptr)
577  val last_cycle_bpu_in_idx = last_cycle_bpu_in_ptr.value
578  val last_cycle_bpu_target = RegNext(bpu_in_resp.getTarget)
579  val last_cycle_cfiIndex = RegNext(bpu_in_resp.cfiIndex)
580  val last_cycle_bpu_in_stage = RegNext(bpu_in_stage)
581
582  def extra_copyNum_for_commitStateQueue = 2
583  val copied_last_cycle_bpu_in = VecInit(Seq.fill(copyNum+extra_copyNum_for_commitStateQueue)(RegNext(bpu_in_fire)))
584  val copied_last_cycle_bpu_in_ptr_for_ftq = VecInit(Seq.fill(extra_copyNum_for_commitStateQueue)(RegNext(bpu_in_resp_ptr)))
585
586  when (last_cycle_bpu_in) {
587    entry_fetch_status(last_cycle_bpu_in_idx) := f_to_send
588    commitStateQueue(last_cycle_bpu_in_idx) := VecInit(Seq.fill(PredictWidth)(c_invalid))
589    cfiIndex_vec(last_cycle_bpu_in_idx) := last_cycle_cfiIndex
590    pred_stage(last_cycle_bpu_in_idx) := last_cycle_bpu_in_stage
591
592    update_target(last_cycle_bpu_in_idx) := last_cycle_bpu_target // TODO: remove this
593    newest_entry_target := last_cycle_bpu_target
594    newest_entry_ptr := last_cycle_bpu_in_ptr
595  }
596
597  // reduce fanout by delay write for a cycle
598  when (RegNext(last_cycle_bpu_in)) {
599    mispredict_vec(RegNext(last_cycle_bpu_in_idx)) := WireInit(VecInit(Seq.fill(PredictWidth)(false.B)))
600  }
601
602  // reduce fanout using copied last_cycle_bpu_in and copied last_cycle_bpu_in_ptr
603  val copied_last_cycle_bpu_in_for_ftq = copied_last_cycle_bpu_in.takeRight(extra_copyNum_for_commitStateQueue)
604  copied_last_cycle_bpu_in_for_ftq.zip(copied_last_cycle_bpu_in_ptr_for_ftq).zipWithIndex.map {
605    case ((in, ptr), i) =>
606      when (in) {
607        val perSetEntries = FtqSize / extra_copyNum_for_commitStateQueue // 32
608        require(FtqSize % extra_copyNum_for_commitStateQueue == 0)
609        for (j <- 0 until perSetEntries) {
610          commitStateQueue(i*perSetEntries+j) := VecInit(Seq.fill(PredictWidth)(c_invalid))
611        }
612      }
613  }
614
615  // num cycle is fixed
616  io.toBackend.newest_entry_ptr := RegNext(newest_entry_ptr)
617  io.toBackend.newest_entry_target := RegNext(newest_entry_target)
618
619
620  bpuPtr := bpuPtr + enq_fire
621  copied_bpu_ptr.map(_ := bpuPtr + enq_fire)
622  when (io.toIfu.req.fire && allowToIfu) {
623    ifuPtr_write := ifuPtrPlus1
624    ifuPtrPlus1_write := ifuPtrPlus2
625    ifuPtrPlus2_write := ifuPtrPlus2 + 1.U
626  }
627
628  // only use ftb result to assign hit status
629  when (bpu_s2_resp.valid) {
630    entry_hit_status(bpu_s2_resp.ftq_idx.value) := Mux(bpu_s2_resp.full_pred.hit, h_hit, h_not_hit)
631  }
632
633
634  io.toIfu.flushFromBpu.s2.valid := bpu_s2_redirect
635  io.toIfu.flushFromBpu.s2.bits := bpu_s2_resp.ftq_idx
636  when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect) {
637    bpuPtr := bpu_s2_resp.ftq_idx + 1.U
638    copied_bpu_ptr.map(_ := bpu_s2_resp.ftq_idx + 1.U)
639    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
640    when (!isBefore(ifuPtr, bpu_s2_resp.ftq_idx)) {
641      ifuPtr_write := bpu_s2_resp.ftq_idx
642      ifuPtrPlus1_write := bpu_s2_resp.ftq_idx + 1.U
643      ifuPtrPlus2_write := bpu_s2_resp.ftq_idx + 2.U
644    }
645  }
646
647  io.toIfu.flushFromBpu.s3.valid := bpu_s3_redirect
648  io.toIfu.flushFromBpu.s3.bits := bpu_s3_resp.ftq_idx
649  when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect) {
650    bpuPtr := bpu_s3_resp.ftq_idx + 1.U
651    copied_bpu_ptr.map(_ := bpu_s3_resp.ftq_idx + 1.U)
652    // only when ifuPtr runs ahead of bpu s2 resp should we recover it
653    when (!isBefore(ifuPtr, bpu_s3_resp.ftq_idx)) {
654      ifuPtr_write := bpu_s3_resp.ftq_idx
655      ifuPtrPlus1_write := bpu_s3_resp.ftq_idx + 1.U
656      ifuPtrPlus2_write := bpu_s3_resp.ftq_idx + 2.U
657    }
658  }
659
660  XSError(isBefore(bpuPtr, ifuPtr) && !isFull(bpuPtr, ifuPtr), "\nifuPtr is before bpuPtr!\n")
661
662  (0 until copyNum).map{i =>
663    XSError(copied_bpu_ptr(i) =/= bpuPtr, "\ncopiedBpuPtr is different from bpuPtr!\n")
664  }
665
666  // ****************************************************************
667  // **************************** to ifu ****************************
668  // ****************************************************************
669  // 0  for ifu, and 1-4 for ICache
670  val bpu_in_bypass_buf = RegEnable(ftq_pc_mem.io.wdata, enable=bpu_in_fire)
671  val copied_bpu_in_bypass_buf = VecInit(Seq.fill(copyNum)(RegEnable(ftq_pc_mem.io.wdata, enable=bpu_in_fire)))
672  val bpu_in_bypass_buf_for_ifu = bpu_in_bypass_buf
673  val bpu_in_bypass_ptr = RegNext(bpu_in_resp_ptr)
674  val last_cycle_to_ifu_fire = RegNext(io.toIfu.req.fire)
675
676  val copied_bpu_in_bypass_ptr = VecInit(Seq.fill(copyNum)(RegNext(bpu_in_resp_ptr)))
677  val copied_last_cycle_to_ifu_fire = VecInit(Seq.fill(copyNum)(RegNext(io.toIfu.req.fire)))
678
679  // read pc and target
680  ftq_pc_mem.io.ifuPtr_w       := ifuPtr_write
681  ftq_pc_mem.io.ifuPtrPlus1_w  := ifuPtrPlus1_write
682  ftq_pc_mem.io.ifuPtrPlus2_w  := ifuPtrPlus2_write
683  ftq_pc_mem.io.commPtr_w      := commPtr_write
684  ftq_pc_mem.io.commPtrPlus1_w := commPtrPlus1_write
685
686
687  io.toIfu.req.bits.ftqIdx := ifuPtr
688
689  val toICachePcBundle = Wire(Vec(copyNum,new Ftq_RF_Components))
690  val toICacheEntryToSend = Wire(Vec(copyNum,Bool()))
691  val toIfuPcBundle = Wire(new Ftq_RF_Components)
692  val entry_is_to_send = WireInit(entry_fetch_status(ifuPtr.value) === f_to_send)
693  val entry_ftq_offset = WireInit(cfiIndex_vec(ifuPtr.value))
694  val entry_next_addr  = Wire(UInt(VAddrBits.W))
695
696  val pc_mem_ifu_ptr_rdata   = VecInit(Seq.fill(copyNum)(RegNext(ftq_pc_mem.io.ifuPtr_rdata)))
697  val pc_mem_ifu_plus1_rdata = VecInit(Seq.fill(copyNum)(RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata)))
698  val diff_entry_next_addr = WireInit(update_target(ifuPtr.value)) //TODO: remove this
699
700  val copied_ifu_plus1_to_send = VecInit(Seq.fill(copyNum)(RegNext(entry_fetch_status(ifuPtrPlus1.value) === f_to_send) || RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1))))
701  val copied_ifu_ptr_to_send   = VecInit(Seq.fill(copyNum)(RegNext(entry_fetch_status(ifuPtr.value) === f_to_send) || RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr)))
702
703  for(i <- 0 until copyNum){
704    when(copied_last_cycle_bpu_in(i) && copied_bpu_in_bypass_ptr(i) === copied_ifu_ptr(i)){
705      toICachePcBundle(i) := copied_bpu_in_bypass_buf(i)
706      toICacheEntryToSend(i)   := true.B
707    }.elsewhen(copied_last_cycle_to_ifu_fire(i)){
708      toICachePcBundle(i) := pc_mem_ifu_plus1_rdata(i)
709      toICacheEntryToSend(i)   := copied_ifu_plus1_to_send(i)
710    }.otherwise{
711      toICachePcBundle(i) := pc_mem_ifu_ptr_rdata(i)
712      toICacheEntryToSend(i)   := copied_ifu_ptr_to_send(i)
713    }
714  }
715
716  // TODO: reconsider target address bypass logic
717  when (last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) {
718    toIfuPcBundle := bpu_in_bypass_buf_for_ifu
719    entry_is_to_send := true.B
720    entry_next_addr := last_cycle_bpu_target
721    entry_ftq_offset := last_cycle_cfiIndex
722    diff_entry_next_addr := last_cycle_bpu_target // TODO: remove this
723  }.elsewhen (last_cycle_to_ifu_fire) {
724    toIfuPcBundle := RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata)
725    entry_is_to_send := RegNext(entry_fetch_status(ifuPtrPlus1.value) === f_to_send) ||
726                        RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1)) // reduce potential bubbles
727    entry_next_addr := Mux(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1),
728                          bpu_in_bypass_buf_for_ifu.startAddr,
729                          Mux(ifuPtr === newest_entry_ptr,
730                            newest_entry_target,
731                            RegNext(ftq_pc_mem.io.ifuPtrPlus2_rdata.startAddr))) // ifuPtr+2
732  }.otherwise {
733    toIfuPcBundle := RegNext(ftq_pc_mem.io.ifuPtr_rdata)
734    entry_is_to_send := RegNext(entry_fetch_status(ifuPtr.value) === f_to_send) ||
735                        RegNext(last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr) // reduce potential bubbles
736    entry_next_addr := Mux(last_cycle_bpu_in && bpu_in_bypass_ptr === (ifuPtrPlus1),
737                          bpu_in_bypass_buf_for_ifu.startAddr,
738                          Mux(ifuPtr === newest_entry_ptr,
739                            newest_entry_target,
740                            RegNext(ftq_pc_mem.io.ifuPtrPlus1_rdata.startAddr))) // ifuPtr+1
741  }
742
743  io.toIfu.req.valid := entry_is_to_send && ifuPtr =/= bpuPtr
744  io.toIfu.req.bits.nextStartAddr := entry_next_addr
745  io.toIfu.req.bits.ftqOffset := entry_ftq_offset
746  io.toIfu.req.bits.fromFtqPcBundle(toIfuPcBundle)
747
748  io.toICache.req.valid := entry_is_to_send && ifuPtr =/= bpuPtr
749  io.toICache.req.bits.readValid.zipWithIndex.map{case(copy, i) => copy := toICacheEntryToSend(i) && copied_ifu_ptr(i) =/= copied_bpu_ptr(i)}
750  io.toICache.req.bits.pcMemRead.zipWithIndex.map{case(copy,i) => copy.fromFtqPcBundle(toICachePcBundle(i))}
751  // io.toICache.req.bits.bypassSelect := last_cycle_bpu_in && bpu_in_bypass_ptr === ifuPtr
752  // io.toICache.req.bits.bpuBypassWrite.zipWithIndex.map{case(bypassWrtie, i) =>
753  //   bypassWrtie.startAddr := bpu_in_bypass_buf.tail(i).startAddr
754  //   bypassWrtie.nextlineStart := bpu_in_bypass_buf.tail(i).nextLineAddr
755  // }
756
757  // TODO: remove this
758  XSError(io.toIfu.req.valid && diff_entry_next_addr =/= entry_next_addr,
759          p"\nifu_req_target wrong! ifuPtr: ${ifuPtr}, entry_next_addr: ${Hexadecimal(entry_next_addr)} diff_entry_next_addr: ${Hexadecimal(diff_entry_next_addr)}\n")
760
761  // when fall through is smaller in value than start address, there must be a false hit
762  when (toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit) {
763    when (io.toIfu.req.fire &&
764      !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) &&
765      !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr)
766    ) {
767      entry_hit_status(ifuPtr.value) := h_false_hit
768      // XSError(true.B, "FTB false hit by fallThroughError, startAddr: %x, fallTHru: %x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
769    }
770    XSDebug(true.B, "fallThruError! start:%x, fallThru:%x\n", io.toIfu.req.bits.startAddr, io.toIfu.req.bits.nextStartAddr)
771  }
772
773  XSPerfAccumulate(f"fall_through_error_to_ifu", toIfuPcBundle.fallThruError && entry_hit_status(ifuPtr.value) === h_hit &&
774    io.toIfu.req.fire && !(bpu_s2_redirect && bpu_s2_resp.ftq_idx === ifuPtr) && !(bpu_s3_redirect && bpu_s3_resp.ftq_idx === ifuPtr))
775
776  val ifu_req_should_be_flushed =
777    io.toIfu.flushFromBpu.shouldFlushByStage2(io.toIfu.req.bits.ftqIdx) ||
778    io.toIfu.flushFromBpu.shouldFlushByStage3(io.toIfu.req.bits.ftqIdx)
779
780    when (io.toIfu.req.fire && !ifu_req_should_be_flushed) {
781      entry_fetch_status(ifuPtr.value) := f_sent
782    }
783
784  // *********************************************************************
785  // **************************** wb from ifu ****************************
786  // *********************************************************************
787  val pdWb = io.fromIfu.pdWb
788  val pds = pdWb.bits.pd
789  val ifu_wb_valid = pdWb.valid
790  val ifu_wb_idx = pdWb.bits.ftqIdx.value
791  // read ports:                                                         commit update
792  val ftq_pd_mem = Module(new SyncDataModuleTemplate(new Ftq_pd_Entry, FtqSize, 1, 1))
793  ftq_pd_mem.io.wen(0) := ifu_wb_valid
794  ftq_pd_mem.io.waddr(0) := pdWb.bits.ftqIdx.value
795  ftq_pd_mem.io.wdata(0).fromPdWb(pdWb.bits)
796
797  val hit_pd_valid = entry_hit_status(ifu_wb_idx) === h_hit && ifu_wb_valid
798  val hit_pd_mispred = hit_pd_valid && pdWb.bits.misOffset.valid
799  val hit_pd_mispred_reg = RegNext(hit_pd_mispred, init=false.B)
800  val pd_reg       = RegEnable(pds,             pdWb.valid)
801  val start_pc_reg = RegEnable(pdWb.bits.pc(0), pdWb.valid)
802  val wb_idx_reg   = RegEnable(ifu_wb_idx,      pdWb.valid)
803
804  when (ifu_wb_valid) {
805    val comm_stq_wen = VecInit(pds.map(_.valid).zip(pdWb.bits.instrRange).map{
806      case (v, inRange) => v && inRange
807    })
808    (commitStateQueue(ifu_wb_idx) zip comm_stq_wen).map{
809      case (qe, v) => when (v) { qe := c_valid }
810    }
811  }
812
813  when (ifu_wb_valid) {
814    ifuWbPtr_write := ifuWbPtr + 1.U
815  }
816
817  ftb_entry_mem.io.raddr.head := ifu_wb_idx
818  val has_false_hit = WireInit(false.B)
819  when (RegNext(hit_pd_valid)) {
820    // check for false hit
821    val pred_ftb_entry = ftb_entry_mem.io.rdata.head
822    val brSlots = pred_ftb_entry.brSlots
823    val tailSlot = pred_ftb_entry.tailSlot
824    // we check cfis that bpu predicted
825
826    // bpu predicted branches but denied by predecode
827    val br_false_hit =
828      brSlots.map{
829        s => s.valid && !(pd_reg(s.offset).valid && pd_reg(s.offset).isBr)
830      }.reduce(_||_) ||
831      (tailSlot.valid && pred_ftb_entry.tailSlot.sharing &&
832        !(pd_reg(tailSlot.offset).valid && pd_reg(tailSlot.offset).isBr))
833
834    val jmpOffset = tailSlot.offset
835    val jmp_pd = pd_reg(jmpOffset)
836    val jal_false_hit = pred_ftb_entry.jmpValid &&
837      ((pred_ftb_entry.isJal  && !(jmp_pd.valid && jmp_pd.isJal)) ||
838       (pred_ftb_entry.isJalr && !(jmp_pd.valid && jmp_pd.isJalr)) ||
839       (pred_ftb_entry.isCall && !(jmp_pd.valid && jmp_pd.isCall)) ||
840       (pred_ftb_entry.isRet  && !(jmp_pd.valid && jmp_pd.isRet))
841      )
842
843    has_false_hit := br_false_hit || jal_false_hit || hit_pd_mispred_reg
844    XSDebug(has_false_hit, "FTB false hit by br or jal or hit_pd, startAddr: %x\n", pdWb.bits.pc(0))
845
846    // assert(!has_false_hit)
847  }
848
849  when (has_false_hit) {
850    entry_hit_status(wb_idx_reg) := h_false_hit
851  }
852
853
854  // **********************************************************************
855  // ***************************** to backend *****************************
856  // **********************************************************************
857  // to backend pc mem / target
858  io.toBackend.pc_mem_wen   := RegNext(last_cycle_bpu_in)
859  io.toBackend.pc_mem_waddr := RegNext(last_cycle_bpu_in_idx)
860  io.toBackend.pc_mem_wdata := RegNext(bpu_in_bypass_buf_for_ifu)
861
862  // *******************************************************************************
863  // **************************** redirect from backend ****************************
864  // *******************************************************************************
865
866  // redirect read cfiInfo, couples to redirectGen s2
867  ftq_redirect_sram.io.ren.init.last := backendRedirect.valid
868  ftq_redirect_sram.io.raddr.init.last := backendRedirect.bits.ftqIdx.value
869
870  ftb_entry_mem.io.raddr.init.last := backendRedirect.bits.ftqIdx.value
871
872  val stage3CfiInfo = ftq_redirect_sram.io.rdata.init.last
873  val fromBackendRedirect = WireInit(backendRedirectReg)
874  val backendRedirectCfi = fromBackendRedirect.bits.cfiUpdate
875  backendRedirectCfi.fromFtqRedirectSram(stage3CfiInfo)
876
877  val r_ftb_entry = ftb_entry_mem.io.rdata.init.last
878  val r_ftqOffset = fromBackendRedirect.bits.ftqOffset
879
880  when (entry_hit_status(fromBackendRedirect.bits.ftqIdx.value) === h_hit) {
881    backendRedirectCfi.shift := PopCount(r_ftb_entry.getBrMaskByOffset(r_ftqOffset)) +&
882      (backendRedirectCfi.pd.isBr && !r_ftb_entry.brIsSaved(r_ftqOffset) &&
883      !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
884
885    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr && (r_ftb_entry.brIsSaved(r_ftqOffset) ||
886        !r_ftb_entry.newBrCanNotInsert(r_ftqOffset))
887  }.otherwise {
888    backendRedirectCfi.shift := (backendRedirectCfi.pd.isBr && backendRedirectCfi.taken).asUInt
889    backendRedirectCfi.addIntoHist := backendRedirectCfi.pd.isBr.asUInt
890  }
891
892
893  // ***************************************************************************
894  // **************************** redirect from ifu ****************************
895  // ***************************************************************************
896  val fromIfuRedirect = WireInit(0.U.asTypeOf(Valid(new Redirect)))
897  fromIfuRedirect.valid := pdWb.valid && pdWb.bits.misOffset.valid && !backendFlush
898  fromIfuRedirect.bits.ftqIdx := pdWb.bits.ftqIdx
899  fromIfuRedirect.bits.ftqOffset := pdWb.bits.misOffset.bits
900  fromIfuRedirect.bits.level := RedirectLevel.flushAfter
901
902  val ifuRedirectCfiUpdate = fromIfuRedirect.bits.cfiUpdate
903  ifuRedirectCfiUpdate.pc := pdWb.bits.pc(pdWb.bits.misOffset.bits)
904  ifuRedirectCfiUpdate.pd := pdWb.bits.pd(pdWb.bits.misOffset.bits)
905  ifuRedirectCfiUpdate.predTaken := cfiIndex_vec(pdWb.bits.ftqIdx.value).valid
906  ifuRedirectCfiUpdate.target := pdWb.bits.target
907  ifuRedirectCfiUpdate.taken := pdWb.bits.cfiOffset.valid
908  ifuRedirectCfiUpdate.isMisPred := pdWb.bits.misOffset.valid
909
910  val ifuRedirectReg = RegNext(fromIfuRedirect, init=0.U.asTypeOf(Valid(new Redirect)))
911  val ifuRedirectToBpu = WireInit(ifuRedirectReg)
912  ifuFlush := fromIfuRedirect.valid || ifuRedirectToBpu.valid
913
914  ftq_redirect_sram.io.ren.head := fromIfuRedirect.valid
915  ftq_redirect_sram.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
916
917  ftb_entry_mem.io.raddr.head := fromIfuRedirect.bits.ftqIdx.value
918
919  val toBpuCfi = ifuRedirectToBpu.bits.cfiUpdate
920  toBpuCfi.fromFtqRedirectSram(ftq_redirect_sram.io.rdata.head)
921  when (ifuRedirectReg.bits.cfiUpdate.pd.isRet) {
922    toBpuCfi.target := toBpuCfi.rasEntry.retAddr
923  }
924
925  // *********************************************************************
926  // **************************** wb from exu ****************************
927  // *********************************************************************
928
929  backendRedirect := io.fromBackend.redirect
930
931  def extractRedirectInfo(wb: Valid[Redirect]) = {
932    val ftqPtr = wb.bits.ftqIdx
933    val ftqOffset = wb.bits.ftqOffset
934    val taken = wb.bits.cfiUpdate.taken
935    val mispred = wb.bits.cfiUpdate.isMisPred
936    (wb.valid, ftqPtr, ftqOffset, taken, mispred)
937  }
938
939  // fix mispredict entry
940  val lastIsMispredict = RegNext(
941    backendRedirect.valid && backendRedirect.bits.level === RedirectLevel.flushAfter, init = false.B
942  )
943
944  def updateCfiInfo(redirect: Valid[Redirect], isBackend: Boolean = true) = {
945    val (r_valid, r_ptr, r_offset, r_taken, r_mispred) = extractRedirectInfo(redirect)
946    val r_idx = r_ptr.value
947    val cfiIndex_bits_wen = r_valid && r_taken && r_offset < cfiIndex_vec(r_idx).bits
948    val cfiIndex_valid_wen = r_valid && r_offset === cfiIndex_vec(r_idx).bits
949    when (cfiIndex_bits_wen || cfiIndex_valid_wen) {
950      cfiIndex_vec(r_idx).valid := cfiIndex_bits_wen || cfiIndex_valid_wen && r_taken
951    }
952    when (cfiIndex_bits_wen) {
953      cfiIndex_vec(r_idx).bits := r_offset
954    }
955    newest_entry_target := redirect.bits.cfiUpdate.target
956    newest_entry_ptr := r_ptr
957    update_target(r_idx) := redirect.bits.cfiUpdate.target // TODO: remove this
958    if (isBackend) {
959      mispredict_vec(r_idx)(r_offset) := r_mispred
960    }
961  }
962
963  when(backendRedirectReg.valid) {
964    updateCfiInfo(backendRedirectReg)
965  }.elsewhen (ifuRedirectToBpu.valid) {
966    updateCfiInfo(ifuRedirectToBpu, isBackend=false)
967  }
968
969  // ***********************************************************************************
970  // **************************** flush ptr and state queue ****************************
971  // ***********************************************************************************
972
973  val redirectVec = VecInit(backendRedirect, fromIfuRedirect)
974
975  // when redirect, we should reset ptrs and status queues
976  when(redirectVec.map(r => r.valid).reduce(_||_)){
977    val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
978    val notIfu = redirectVec.dropRight(1).map(r => r.valid).reduce(_||_)
979    val (idx, offset, flushItSelf) = (r.ftqIdx, r.ftqOffset, RedirectLevel.flushItself(r.level))
980    val next = idx + 1.U
981    bpuPtr := next
982    copied_bpu_ptr.map(_ := next)
983    ifuPtr_write := next
984    ifuWbPtr_write := next
985    ifuPtrPlus1_write := idx + 2.U
986    ifuPtrPlus2_write := idx + 3.U
987    when (notIfu) {
988      commitStateQueue(idx.value).zipWithIndex.foreach({ case (s, i) =>
989        when(i.U > offset || i.U === offset && flushItSelf){
990          s := c_invalid
991        }
992      })
993    }
994  }
995
996  // only the valid bit is actually needed
997  io.toIfu.redirect.bits    := backendRedirect.bits
998  io.toIfu.redirect.valid   := stage2Flush
999
1000  // commit
1001  for (c <- io.fromBackend.rob_commits) {
1002    when(c.valid) {
1003      commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset) := c_commited
1004      // TODO: remove this
1005      // For instruction fusions, we also update the next instruction
1006      when (c.bits.commitType === 4.U) {
1007        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 1.U) := c_commited
1008      }.elsewhen(c.bits.commitType === 5.U) {
1009        commitStateQueue(c.bits.ftqIdx.value)(c.bits.ftqOffset + 2.U) := c_commited
1010      }.elsewhen(c.bits.commitType === 6.U) {
1011        val index = (c.bits.ftqIdx + 1.U).value
1012        commitStateQueue(index)(0) := c_commited
1013      }.elsewhen(c.bits.commitType === 7.U) {
1014        val index = (c.bits.ftqIdx + 1.U).value
1015        commitStateQueue(index)(1) := c_commited
1016      }
1017    }
1018  }
1019
1020  // ****************************************************************
1021  // **************************** to bpu ****************************
1022  // ****************************************************************
1023
1024  io.toBpu.redirect <> Mux(fromBackendRedirect.valid, fromBackendRedirect, ifuRedirectToBpu)
1025
1026  val may_have_stall_from_bpu = Wire(Bool())
1027  val bpu_ftb_update_stall = RegInit(0.U(2.W)) // 2-cycle stall, so we need 3 states
1028  may_have_stall_from_bpu := bpu_ftb_update_stall =/= 0.U
1029  val canCommit = commPtr =/= ifuWbPtr && !may_have_stall_from_bpu &&
1030    Cat(commitStateQueue(commPtr.value).map(s => {
1031      s === c_invalid || s === c_commited
1032    })).andR()
1033
1034  // commit reads
1035  val commit_pc_bundle = RegNext(ftq_pc_mem.io.commPtr_rdata)
1036  val commit_target =
1037    Mux(RegNext(commPtr === newest_entry_ptr),
1038      RegNext(newest_entry_target),
1039      RegNext(ftq_pc_mem.io.commPtrPlus1_rdata.startAddr))
1040  ftq_pd_mem.io.raddr.last := commPtr.value
1041  val commit_pd = ftq_pd_mem.io.rdata.last
1042  ftq_redirect_sram.io.ren.last := canCommit
1043  ftq_redirect_sram.io.raddr.last := commPtr.value
1044  val commit_spec_meta = ftq_redirect_sram.io.rdata.last
1045  ftq_meta_1r_sram.io.ren(0) := canCommit
1046  ftq_meta_1r_sram.io.raddr(0) := commPtr.value
1047  val commit_meta = ftq_meta_1r_sram.io.rdata(0)
1048  ftb_entry_mem.io.raddr.last := commPtr.value
1049  val commit_ftb_entry = ftb_entry_mem.io.rdata.last
1050
1051  // need one cycle to read mem and srams
1052  val do_commit_ptr = RegNext(commPtr)
1053  val do_commit = RegNext(canCommit, init=false.B)
1054  when (canCommit) {
1055    commPtr_write := commPtrPlus1
1056    commPtrPlus1_write := commPtrPlus1 + 1.U
1057  }
1058  val commit_state = RegNext(commitStateQueue(commPtr.value))
1059  val can_commit_cfi = WireInit(cfiIndex_vec(commPtr.value))
1060  when (commitStateQueue(commPtr.value)(can_commit_cfi.bits) =/= c_commited) {
1061    can_commit_cfi.valid := false.B
1062  }
1063  val commit_cfi = RegNext(can_commit_cfi)
1064
1065  val commit_mispredict = VecInit((RegNext(mispredict_vec(commPtr.value)) zip commit_state).map {
1066    case (mis, state) => mis && state === c_commited
1067  })
1068  val can_commit_hit = entry_hit_status(commPtr.value)
1069  val commit_hit = RegNext(can_commit_hit)
1070  val diff_commit_target = RegNext(update_target(commPtr.value)) // TODO: remove this
1071  val commit_stage = RegNext(pred_stage(commPtr.value))
1072  val commit_valid = commit_hit === h_hit || commit_cfi.valid // hit or taken
1073
1074  val to_bpu_hit = can_commit_hit === h_hit || can_commit_hit === h_false_hit
1075  switch (bpu_ftb_update_stall) {
1076    is (0.U) {
1077      when (can_commit_cfi.valid && !to_bpu_hit && canCommit) {
1078        bpu_ftb_update_stall := 2.U // 2-cycle stall
1079      }
1080    }
1081    is (2.U) {
1082      bpu_ftb_update_stall := 1.U
1083    }
1084    is (1.U) {
1085      bpu_ftb_update_stall := 0.U
1086    }
1087    is (3.U) {
1088      XSError(true.B, "bpu_ftb_update_stall should be 0, 1 or 2")
1089    }
1090  }
1091
1092  // TODO: remove this
1093  XSError(do_commit && diff_commit_target =/= commit_target, "\ncommit target should be the same as update target\n")
1094
1095  io.toBpu.update := DontCare
1096  io.toBpu.update.valid := commit_valid && do_commit
1097  val update = io.toBpu.update.bits
1098  update.false_hit   := commit_hit === h_false_hit
1099  update.pc          := commit_pc_bundle.startAddr
1100  update.meta        := commit_meta.meta
1101  update.full_target := commit_target
1102  update.from_stage  := commit_stage
1103  update.fromFtqRedirectSram(commit_spec_meta)
1104
1105  val commit_real_hit = commit_hit === h_hit
1106  val update_ftb_entry = update.ftb_entry
1107
1108  val ftbEntryGen = Module(new FTBEntryGen).io
1109  ftbEntryGen.start_addr     := commit_pc_bundle.startAddr
1110  ftbEntryGen.old_entry      := commit_ftb_entry
1111  ftbEntryGen.pd             := commit_pd
1112  ftbEntryGen.cfiIndex       := commit_cfi
1113  ftbEntryGen.target         := commit_target
1114  ftbEntryGen.hit            := commit_real_hit
1115  ftbEntryGen.mispredict_vec := commit_mispredict
1116
1117  update_ftb_entry         := ftbEntryGen.new_entry
1118  update.new_br_insert_pos := ftbEntryGen.new_br_insert_pos
1119  update.mispred_mask      := ftbEntryGen.mispred_mask
1120  update.old_entry         := ftbEntryGen.is_old_entry
1121  update.pred_hit          := commit_hit === h_hit || commit_hit === h_false_hit
1122
1123  update.is_minimal := false.B
1124  update.full_pred.fromFtbEntry(ftbEntryGen.new_entry, update.pc)
1125  update.full_pred.br_taken_mask  := ftbEntryGen.taken_mask
1126  update.full_pred.jalr_target := commit_target
1127  update.full_pred.hit := true.B
1128  when (update.full_pred.is_jalr) {
1129    update.full_pred.targets.last := commit_target
1130  }
1131
1132  // ****************************************************************
1133  // *********************** to prefetch ****************************
1134  // ****************************************************************
1135
1136  ftq_pc_mem.io.other_raddrs(0) := DontCare
1137  if(cacheParams.hasPrefetch){
1138    val prefetchPtr = RegInit(FtqPtr(false.B, 0.U))
1139    val diff_prefetch_addr = WireInit(update_target(prefetchPtr.value)) //TODO: remove this
1140
1141    prefetchPtr := prefetchPtr + io.toPrefetch.req.fire()
1142
1143    ftq_pc_mem.io.other_raddrs(0) := prefetchPtr.value
1144
1145    when (bpu_s2_resp.valid && bpu_s2_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s2_resp.ftq_idx)) {
1146      prefetchPtr := bpu_s2_resp.ftq_idx
1147    }
1148
1149    when (bpu_s3_resp.valid && bpu_s3_resp.hasRedirect && !isBefore(prefetchPtr, bpu_s3_resp.ftq_idx)) {
1150      prefetchPtr := bpu_s3_resp.ftq_idx
1151      // XSError(true.B, "\ns3_redirect mechanism not implemented!\n")
1152    }
1153
1154
1155    val prefetch_is_to_send = WireInit(entry_fetch_status(prefetchPtr.value) === f_to_send)
1156    val prefetch_addr = Wire(UInt(VAddrBits.W))
1157
1158    when (last_cycle_bpu_in && bpu_in_bypass_ptr === prefetchPtr) {
1159      prefetch_is_to_send := true.B
1160      prefetch_addr := last_cycle_bpu_target
1161      diff_prefetch_addr := last_cycle_bpu_target // TODO: remove this
1162    }.otherwise{
1163      prefetch_addr := RegNext( ftq_pc_mem.io.other_rdatas(0).startAddr)
1164    }
1165    io.toPrefetch.req.valid := prefetchPtr =/= bpuPtr && prefetch_is_to_send
1166    io.toPrefetch.req.bits.target := prefetch_addr
1167
1168    when(redirectVec.map(r => r.valid).reduce(_||_)){
1169      val r = PriorityMux(redirectVec.map(r => (r.valid -> r.bits)))
1170      val next = r.ftqIdx + 1.U
1171      prefetchPtr := next
1172    }
1173
1174    // TODO: remove this
1175    // XSError(io.toPrefetch.req.valid && diff_prefetch_addr =/= prefetch_addr,
1176    //         f"\nprefetch_req_target wrong! prefetchPtr: ${prefetchPtr}, prefetch_addr: ${Hexadecimal(prefetch_addr)} diff_prefetch_addr: ${Hexadecimal(diff_prefetch_addr)}\n")
1177
1178
1179    XSError(isBefore(bpuPtr, prefetchPtr) && !isFull(bpuPtr, prefetchPtr), "\nprefetchPtr is before bpuPtr!\n")
1180    XSError(isBefore(prefetchPtr, ifuPtr) && !isFull(ifuPtr, prefetchPtr), "\nifuPtr is before prefetchPtr!\n")
1181  }
1182  else {
1183    io.toPrefetch.req <> DontCare
1184  }
1185
1186  // ******************************************************************************
1187  // **************************** commit perf counters ****************************
1188  // ******************************************************************************
1189
1190  val commit_inst_mask    = VecInit(commit_state.map(c => c === c_commited && do_commit)).asUInt
1191  val commit_mispred_mask = commit_mispredict.asUInt
1192  val commit_not_mispred_mask = ~commit_mispred_mask
1193
1194  val commit_br_mask = commit_pd.brMask.asUInt
1195  val commit_jmp_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.jmpInfo.valid.asTypeOf(UInt(1.W)))
1196  val commit_cfi_mask = (commit_br_mask | commit_jmp_mask)
1197
1198  val mbpInstrs = commit_inst_mask & commit_cfi_mask
1199
1200  val mbpRights = mbpInstrs & commit_not_mispred_mask
1201  val mbpWrongs = mbpInstrs & commit_mispred_mask
1202
1203  io.bpuInfo.bpRight := PopCount(mbpRights)
1204  io.bpuInfo.bpWrong := PopCount(mbpWrongs)
1205
1206  // Cfi Info
1207  for (i <- 0 until PredictWidth) {
1208    val pc = commit_pc_bundle.startAddr + (i * instBytes).U
1209    val v = commit_state(i) === c_commited
1210    val isBr = commit_pd.brMask(i)
1211    val isJmp = commit_pd.jmpInfo.valid && commit_pd.jmpOffset === i.U
1212    val isCfi = isBr || isJmp
1213    val isTaken = commit_cfi.valid && commit_cfi.bits === i.U
1214    val misPred = commit_mispredict(i)
1215    // val ghist = commit_spec_meta.ghist.predHist
1216    val histPtr = commit_spec_meta.histPtr
1217    val predCycle = commit_meta.meta(63, 0)
1218    val target = commit_target
1219
1220    val brIdx = OHToUInt(Reverse(Cat(update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U})))
1221    val inFtbEntry = update_ftb_entry.brValids.zip(update_ftb_entry.brOffset).map{case(v, offset) => v && offset === i.U}.reduce(_||_)
1222    val addIntoHist = ((commit_hit === h_hit) && inFtbEntry) || ((!(commit_hit === h_hit) && i.U === commit_cfi.bits && isBr && commit_cfi.valid))
1223    XSDebug(v && do_commit && isCfi, p"cfi_update: isBr(${isBr}) pc(${Hexadecimal(pc)}) " +
1224    p"taken(${isTaken}) mispred(${misPred}) cycle($predCycle) hist(${histPtr.value}) " +
1225    p"startAddr(${Hexadecimal(commit_pc_bundle.startAddr)}) AddIntoHist(${addIntoHist}) " +
1226    p"brInEntry(${inFtbEntry}) brIdx(${brIdx}) target(${Hexadecimal(target)})\n")
1227  }
1228
1229  val enq = io.fromBpu.resp
1230  val perf_redirect = backendRedirect
1231
1232  XSPerfAccumulate("entry", validEntries)
1233  XSPerfAccumulate("bpu_to_ftq_stall", enq.valid && !enq.ready)
1234  XSPerfAccumulate("mispredictRedirect", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level)
1235  XSPerfAccumulate("replayRedirect", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level))
1236  XSPerfAccumulate("predecodeRedirect", fromIfuRedirect.valid)
1237
1238  XSPerfAccumulate("to_ifu_bubble", io.toIfu.req.ready && !io.toIfu.req.valid)
1239
1240  XSPerfAccumulate("to_ifu_stall", io.toIfu.req.valid && !io.toIfu.req.ready)
1241  XSPerfAccumulate("from_bpu_real_bubble", !enq.valid && enq.ready && allowBpuIn)
1242  XSPerfAccumulate("bpu_to_ifu_bubble", bpuPtr === ifuPtr)
1243
1244  val from_bpu = io.fromBpu.resp.bits
1245  def in_entry_len_map_gen(resp: BranchPredictionBundle)(stage: String) = {
1246    assert(!resp.is_minimal)
1247    val entry_len = (resp.ftb_entry.getFallThrough(resp.pc) - resp.pc) >> instOffsetBits
1248    val entry_len_recording_vec = (1 to PredictWidth+1).map(i => entry_len === i.U)
1249    val entry_len_map = (1 to PredictWidth+1).map(i =>
1250      f"${stage}_ftb_entry_len_$i" -> (entry_len_recording_vec(i-1) && resp.valid)
1251    ).foldLeft(Map[String, UInt]())(_+_)
1252    entry_len_map
1253  }
1254  val s2_entry_len_map = in_entry_len_map_gen(from_bpu.s2)("s2")
1255  val s3_entry_len_map = in_entry_len_map_gen(from_bpu.s3)("s3")
1256
1257  val to_ifu = io.toIfu.req.bits
1258
1259
1260
1261  val commit_num_inst_recording_vec = (1 to PredictWidth).map(i => PopCount(commit_inst_mask) === i.U)
1262  val commit_num_inst_map = (1 to PredictWidth).map(i =>
1263    f"commit_num_inst_$i" -> (commit_num_inst_recording_vec(i-1) && do_commit)
1264  ).foldLeft(Map[String, UInt]())(_+_)
1265
1266
1267
1268  val commit_jal_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJal.asTypeOf(UInt(1.W)))
1269  val commit_jalr_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasJalr.asTypeOf(UInt(1.W)))
1270  val commit_call_mask = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasCall.asTypeOf(UInt(1.W)))
1271  val commit_ret_mask  = UIntToOH(commit_pd.jmpOffset) & Fill(PredictWidth, commit_pd.hasRet.asTypeOf(UInt(1.W)))
1272
1273
1274  val mbpBRights = mbpRights & commit_br_mask
1275  val mbpJRights = mbpRights & commit_jal_mask
1276  val mbpIRights = mbpRights & commit_jalr_mask
1277  val mbpCRights = mbpRights & commit_call_mask
1278  val mbpRRights = mbpRights & commit_ret_mask
1279
1280  val mbpBWrongs = mbpWrongs & commit_br_mask
1281  val mbpJWrongs = mbpWrongs & commit_jal_mask
1282  val mbpIWrongs = mbpWrongs & commit_jalr_mask
1283  val mbpCWrongs = mbpWrongs & commit_call_mask
1284  val mbpRWrongs = mbpWrongs & commit_ret_mask
1285
1286  val commit_pred_stage = RegNext(pred_stage(commPtr.value))
1287
1288  def pred_stage_map(src: UInt, name: String) = {
1289    (0 until numBpStages).map(i =>
1290      f"${name}_stage_${i+1}" -> PopCount(src.asBools.map(_ && commit_pred_stage === BP_STAGES(i)))
1291    ).foldLeft(Map[String, UInt]())(_+_)
1292  }
1293
1294  val mispred_stage_map      = pred_stage_map(mbpWrongs,  "mispredict")
1295  val br_mispred_stage_map   = pred_stage_map(mbpBWrongs, "br_mispredict")
1296  val jalr_mispred_stage_map = pred_stage_map(mbpIWrongs, "jalr_mispredict")
1297  val correct_stage_map      = pred_stage_map(mbpRights,  "correct")
1298  val br_correct_stage_map   = pred_stage_map(mbpBRights, "br_correct")
1299  val jalr_correct_stage_map = pred_stage_map(mbpIRights, "jalr_correct")
1300
1301  val update_valid = io.toBpu.update.valid
1302  def u(cond: Bool) = update_valid && cond
1303  val ftb_false_hit = u(update.false_hit)
1304  // assert(!ftb_false_hit)
1305  val ftb_hit = u(commit_hit === h_hit)
1306
1307  val ftb_new_entry = u(ftbEntryGen.is_init_entry)
1308  val ftb_new_entry_only_br = ftb_new_entry && !update_ftb_entry.jmpValid
1309  val ftb_new_entry_only_jmp = ftb_new_entry && !update_ftb_entry.brValids(0)
1310  val ftb_new_entry_has_br_and_jmp = ftb_new_entry && update_ftb_entry.brValids(0) && update_ftb_entry.jmpValid
1311
1312  val ftb_old_entry = u(ftbEntryGen.is_old_entry)
1313
1314  val ftb_modified_entry = u(ftbEntryGen.is_new_br || ftbEntryGen.is_jalr_target_modified || ftbEntryGen.is_always_taken_modified)
1315  val ftb_modified_entry_new_br = u(ftbEntryGen.is_new_br)
1316  val ftb_modified_entry_jalr_target_modified = u(ftbEntryGen.is_jalr_target_modified)
1317  val ftb_modified_entry_br_full = ftb_modified_entry && ftbEntryGen.is_br_full
1318  val ftb_modified_entry_always_taken = ftb_modified_entry && ftbEntryGen.is_always_taken_modified
1319
1320  val ftb_entry_len = (ftbEntryGen.new_entry.getFallThrough(update.pc) - update.pc) >> instOffsetBits
1321  val ftb_entry_len_recording_vec = (1 to PredictWidth+1).map(i => ftb_entry_len === i.U)
1322  val ftb_init_entry_len_map = (1 to PredictWidth+1).map(i =>
1323    f"ftb_init_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_new_entry)
1324  ).foldLeft(Map[String, UInt]())(_+_)
1325  val ftb_modified_entry_len_map = (1 to PredictWidth+1).map(i =>
1326    f"ftb_modified_entry_len_$i" -> (ftb_entry_len_recording_vec(i-1) && ftb_modified_entry)
1327  ).foldLeft(Map[String, UInt]())(_+_)
1328
1329  val ftq_occupancy_map = (0 to FtqSize).map(i =>
1330    f"ftq_has_entry_$i" ->( validEntries === i.U)
1331  ).foldLeft(Map[String, UInt]())(_+_)
1332
1333  val perfCountsMap = Map(
1334    "BpInstr" -> PopCount(mbpInstrs),
1335    "BpBInstr" -> PopCount(mbpBRights | mbpBWrongs),
1336    "BpRight"  -> PopCount(mbpRights),
1337    "BpWrong"  -> PopCount(mbpWrongs),
1338    "BpBRight" -> PopCount(mbpBRights),
1339    "BpBWrong" -> PopCount(mbpBWrongs),
1340    "BpJRight" -> PopCount(mbpJRights),
1341    "BpJWrong" -> PopCount(mbpJWrongs),
1342    "BpIRight" -> PopCount(mbpIRights),
1343    "BpIWrong" -> PopCount(mbpIWrongs),
1344    "BpCRight" -> PopCount(mbpCRights),
1345    "BpCWrong" -> PopCount(mbpCWrongs),
1346    "BpRRight" -> PopCount(mbpRRights),
1347    "BpRWrong" -> PopCount(mbpRWrongs),
1348
1349    "ftb_false_hit"                -> PopCount(ftb_false_hit),
1350    "ftb_hit"                      -> PopCount(ftb_hit),
1351    "ftb_new_entry"                -> PopCount(ftb_new_entry),
1352    "ftb_new_entry_only_br"        -> PopCount(ftb_new_entry_only_br),
1353    "ftb_new_entry_only_jmp"       -> PopCount(ftb_new_entry_only_jmp),
1354    "ftb_new_entry_has_br_and_jmp" -> PopCount(ftb_new_entry_has_br_and_jmp),
1355    "ftb_old_entry"                -> PopCount(ftb_old_entry),
1356    "ftb_modified_entry"           -> PopCount(ftb_modified_entry),
1357    "ftb_modified_entry_new_br"    -> PopCount(ftb_modified_entry_new_br),
1358    "ftb_jalr_target_modified"     -> PopCount(ftb_modified_entry_jalr_target_modified),
1359    "ftb_modified_entry_br_full"   -> PopCount(ftb_modified_entry_br_full),
1360    "ftb_modified_entry_always_taken" -> PopCount(ftb_modified_entry_always_taken)
1361  ) ++ ftb_init_entry_len_map ++ ftb_modified_entry_len_map ++ s2_entry_len_map ++
1362  s3_entry_len_map ++ commit_num_inst_map ++ ftq_occupancy_map ++
1363  mispred_stage_map ++ br_mispred_stage_map ++ jalr_mispred_stage_map ++
1364  correct_stage_map ++ br_correct_stage_map ++ jalr_correct_stage_map
1365
1366  for((key, value) <- perfCountsMap) {
1367    XSPerfAccumulate(key, value)
1368  }
1369
1370  // --------------------------- Debug --------------------------------
1371  // XSDebug(enq_fire, p"enq! " + io.fromBpu.resp.bits.toPrintable)
1372  XSDebug(io.toIfu.req.fire, p"fire to ifu " + io.toIfu.req.bits.toPrintable)
1373  XSDebug(do_commit, p"deq! [ptr] $do_commit_ptr\n")
1374  XSDebug(true.B, p"[bpuPtr] $bpuPtr, [ifuPtr] $ifuPtr, [ifuWbPtr] $ifuWbPtr [commPtr] $commPtr\n")
1375  XSDebug(true.B, p"[in] v:${io.fromBpu.resp.valid} r:${io.fromBpu.resp.ready} " +
1376    p"[out] v:${io.toIfu.req.valid} r:${io.toIfu.req.ready}\n")
1377  XSDebug(do_commit, p"[deq info] cfiIndex: $commit_cfi, $commit_pc_bundle, target: ${Hexadecimal(commit_target)}\n")
1378
1379  //   def ubtbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1380  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1381  //       case (((valid, pd), ans), taken) =>
1382  //       Mux(valid && pd.isBr,
1383  //         isWrong ^ Mux(ans.hit.asBool,
1384  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1385  //           !taken),
1386  //         !taken),
1387  //       false.B)
1388  //     }
1389  //   }
1390
1391  //   def btbCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1392  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1393  //       case (((valid, pd), ans), taken) =>
1394  //       Mux(valid && pd.isBr,
1395  //         isWrong ^ Mux(ans.hit.asBool,
1396  //           Mux(ans.taken.asBool, taken && ans.target === commitEntry.target,
1397  //           !taken),
1398  //         !taken),
1399  //       false.B)
1400  //     }
1401  //   }
1402
1403  //   def tageCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1404  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1405  //       case (((valid, pd), ans), taken) =>
1406  //       Mux(valid && pd.isBr,
1407  //         isWrong ^ (ans.taken.asBool === taken),
1408  //       false.B)
1409  //     }
1410  //   }
1411
1412  //   def loopCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1413  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1414  //       case (((valid, pd), ans), taken) =>
1415  //       Mux(valid && (pd.isBr) && ans.hit.asBool,
1416  //         isWrong ^ (!taken),
1417  //           false.B)
1418  //     }
1419  //   }
1420
1421  //   def rasCheck(commit: FtqEntry, predAns: Seq[PredictorAnswer], isWrong: Bool) = {
1422  //     commit.valids.zip(commit.pd).zip(predAns).zip(commit.takens).map {
1423  //       case (((valid, pd), ans), taken) =>
1424  //       Mux(valid && pd.isRet.asBool /*&& taken*/ && ans.hit.asBool,
1425  //         isWrong ^ (ans.target === commitEntry.target),
1426  //           false.B)
1427  //     }
1428  //   }
1429
1430  //   val ubtbRights = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), false.B)
1431  //   val ubtbWrongs = ubtbCheck(commitEntry, commitEntry.metas.map(_.ubtbAns), true.B)
1432  //   // btb and ubtb pred jal and jalr as well
1433  //   val btbRights = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), false.B)
1434  //   val btbWrongs = btbCheck(commitEntry, commitEntry.metas.map(_.btbAns), true.B)
1435  //   val tageRights = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), false.B)
1436  //   val tageWrongs = tageCheck(commitEntry, commitEntry.metas.map(_.tageAns), true.B)
1437
1438  //   val loopRights = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), false.B)
1439  //   val loopWrongs = loopCheck(commitEntry, commitEntry.metas.map(_.loopAns), true.B)
1440
1441  //   val rasRights = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), false.B)
1442  //   val rasWrongs = rasCheck(commitEntry, commitEntry.metas.map(_.rasAns), true.B)
1443
1444  val perfEvents = Seq(
1445    ("bpu_s2_redirect        ", bpu_s2_redirect                                                             ),
1446    ("bpu_s3_redirect        ", bpu_s3_redirect                                                             ),
1447    ("bpu_to_ftq_stall       ", enq.valid && ~enq.ready                                                     ),
1448    ("mispredictRedirect     ", perf_redirect.valid && RedirectLevel.flushAfter === perf_redirect.bits.level),
1449    ("replayRedirect         ", perf_redirect.valid && RedirectLevel.flushItself(perf_redirect.bits.level)  ),
1450    ("predecodeRedirect      ", fromIfuRedirect.valid                                                       ),
1451    ("to_ifu_bubble          ", io.toIfu.req.ready && !io.toIfu.req.valid                                   ),
1452    ("from_bpu_real_bubble   ", !enq.valid && enq.ready && allowBpuIn                                       ),
1453    ("BpInstr                ", PopCount(mbpInstrs)                                                         ),
1454    ("BpBInstr               ", PopCount(mbpBRights | mbpBWrongs)                                           ),
1455    ("BpRight                ", PopCount(mbpRights)                                                         ),
1456    ("BpWrong                ", PopCount(mbpWrongs)                                                         ),
1457    ("BpBRight               ", PopCount(mbpBRights)                                                        ),
1458    ("BpBWrong               ", PopCount(mbpBWrongs)                                                        ),
1459    ("BpJRight               ", PopCount(mbpJRights)                                                        ),
1460    ("BpJWrong               ", PopCount(mbpJWrongs)                                                        ),
1461    ("BpIRight               ", PopCount(mbpIRights)                                                        ),
1462    ("BpIWrong               ", PopCount(mbpIWrongs)                                                        ),
1463    ("BpCRight               ", PopCount(mbpCRights)                                                        ),
1464    ("BpCWrong               ", PopCount(mbpCWrongs)                                                        ),
1465    ("BpRRight               ", PopCount(mbpRRights)                                                        ),
1466    ("BpRWrong               ", PopCount(mbpRWrongs)                                                        ),
1467    ("ftb_false_hit          ", PopCount(ftb_false_hit)                                                     ),
1468    ("ftb_hit                ", PopCount(ftb_hit)                                                           ),
1469  )
1470  generatePerfEvent()
1471}